From be330549738003aa1508604471dbd9d0096ab1d0 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Wed, 23 Nov 2022 22:54:59 +0530 Subject: [PATCH 01/39] Possible Solution to Indentation Issue --- core/pkg/fixhandler/datastructures.go | 6 + core/pkg/fixhandler/fixhandler.go | 29 +-- core/pkg/fixhandler/yamlhandler.go | 286 ++++++++++++++++++++++++++ 3 files changed, 295 insertions(+), 26 deletions(-) create mode 100644 core/pkg/fixhandler/yamlhandler.go diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index 6b2601ae..a20fb09c 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -20,3 +20,9 @@ type ResourceFixInfo struct { Resource *reporthandling.Resource FilePath string } + +// LineAndContentToAdd is a struct that holds the information about where to insert the new changes in the existing yaml file +type LineAndContentToAdd struct { + Line int + Content string +} diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 26005b20..bfb6cc49 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -216,32 +216,9 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str } func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError error) { - var completedSuccessfully bool - writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(filePath) - out, err := writeInPlaceHandler.CreateTempFile() - if err != nil { - return fmt.Errorf("unable to create a tmp file for in-place YAML update: %s", err) - } - - defer func() { - if cmdError == nil { - cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) - } - }() - - encoder := yqlib.NewYamlEncoder(2, false, yqlib.ConfiguredYamlPreferences) - - printer := yqlib.NewPrinter(encoder, yqlib.NewSinglePrinterWriter(out)) - allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator() - - preferences := yqlib.ConfiguredYamlPreferences - preferences.EvaluateTogether = true - decoder := yqlib.NewYamlDecoder(preferences) - - err = allAtOnceEvaluator.EvaluateFiles(yamlExpression, []string{filePath}, printer, decoder) - - completedSuccessfully = err == nil - + fixedYamlNode := getFixedYamlNode(filePath, yamlExpression) + lineAndContentsToAdd := getLineAndContentToAdd(&fixedYamlNode) + err := addFixesToFile(filePath, *lineAndContentsToAdd) return err } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go new file mode 100644 index 00000000..e83417e5 --- /dev/null +++ b/core/pkg/fixhandler/yamlhandler.go @@ -0,0 +1,286 @@ +package fixhandler + +import ( + "bufio" + "bytes" + "container/list" + "errors" + "fmt" + "io" + "log" + "os" + "strings" + "yqlib" + + "gopkg.in/yaml.v3" +) + +func getFixedYamlNode(filePath, yamlExpression string) yaml.Node { + preferences := yqlib.ConfiguredYamlPreferences + preferences.EvaluateTogether = true + decoder := yqlib.NewYamlDecoder(preferences) + + var allDocuments = list.New() + reader, err := readStream(filePath) + if err != nil { + return yaml.Node{} + } + + fileDocuments, err := readDocuments(reader, filePath, 0, decoder) + if err != nil { + return yaml.Node{} + } + allDocuments.PushBackList(fileDocuments) + + if allDocuments.Len() == 0 { + candidateNode := &yqlib.CandidateNode{ + Document: 0, + Filename: "", + Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, + FileIndex: 0, + LeadingContent: "", + } + allDocuments.PushBack(candidateNode) + } + + allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator() + + matches, _ := allAtOnceEvaluator.EvaluateCandidateNodes(yamlExpression, allDocuments) + + return *matches.Front().Value.(*yqlib.CandidateNode).Node +} + +func readStream(filename string) (io.Reader, error) { + var reader *bufio.Reader + if filename == "-" { + reader = bufio.NewReader(os.Stdin) + } else { + // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, + // and ensuring that it's not possible to give a path to a file outside thar directory. + file, err := os.Open(filename) // #nosec + if err != nil { + return nil, err + } + reader = bufio.NewReader(file) + } + return reader, nil + +} + +func readDocuments(reader io.Reader, filename string, fileIndex int, decoder yqlib.Decoder) (*list.List, error) { + err := decoder.Init(reader) + if err != nil { + return nil, err + } + inputList := list.New() + var currentIndex uint + + for { + candidateNode, errorReading := decoder.Decode() + + if errors.Is(errorReading, io.EOF) { + switch reader := reader.(type) { + case *os.File: + safelyCloseFile(reader) + } + return inputList, nil + } else if errorReading != nil { + return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading) + } + candidateNode.Document = currentIndex + candidateNode.Filename = filename + candidateNode.FileIndex = fileIndex + candidateNode.EvaluateTogether = true + + inputList.PushBack(candidateNode) + + currentIndex = currentIndex + 1 + } +} + +func safelyCloseFile(file *os.File) { + err := file.Close() + if err != nil { + fmt.Println("Error Closing File") + } +} + +func getLineAndContentToAdd(node *yaml.Node) *[]LineAndContentToAdd { + contentToAdd := make([]LineAndContentToAdd, 0) + getLineAndContentToAddHelper(0, node, &contentToAdd) + return &contentToAdd +} + +func getLineAndContentToAddHelper(nodeIdx int, parentNode *yaml.Node, contentToAdd *[]LineAndContentToAdd) { + node := parentNode.Content[nodeIdx] + var content string + var err error + if node.Line == 0 && node.Column == 0 { + if parentNode.Kind == yaml.MappingNode { + if nodeIdx%2 != 0 { + return + } + } + content, err = enocodeIntoYaml(parentNode, nodeIdx) + + if err != nil { + fmt.Println("Cannot Encode into YAML") + } + + indentationSpacesBeforeContent := parentNode.Column - 1 + + content = addIndentationToContent(content, indentationSpacesBeforeContent) + + // Getting the line to add content after. Add directly after the left Sibling. + var lineToAddAfter int + for idx := nodeIdx - 1; idx >= 0; idx-- { + if parentNode.Content[idx].Line != 0 { + lineToAddAfter = getEndingLine(idx, parentNode) + } + } + lineAndContentToAdd := LineAndContentToAdd{ + Line: lineToAddAfter, + Content: content, + } + *contentToAdd = append(*contentToAdd, lineAndContentToAdd) + } + + for index, _ := range node.Content { + getLineAndContentToAddHelper(index, node, contentToAdd) + } +} + +func enocodeIntoYaml(parentNode *yaml.Node, nodeIdx int) (string, error) { + if parentNode.Kind == yaml.MappingNode { + content := make([]*yaml.Node, 0) + content = append(content, parentNode.Content[nodeIdx], parentNode.Content[nodeIdx+1]) + parentForContent := yaml.Node{ + Kind: yaml.MappingNode, + Content: content, + } + buf := new(bytes.Buffer) + encoder := yaml.NewEncoder(buf) + errorEncoding := encoder.Encode(parentForContent) + if errorEncoding != nil { + return "", fmt.Errorf("Error debugging node, %v", errorEncoding.Error()) + } + errorClosingEncoder := encoder.Close() + if errorClosingEncoder != nil { + return "", fmt.Errorf("Error closing encoder: ", errorClosingEncoder.Error()) + } + return fmt.Sprintf(`%v`, buf.String()), nil + } + + return "", nil +} + +func addIndentationToContent(content string, indentationSpacesBeforeContent int) string { + indentedContent := "" + indentSpaces := strings.Repeat(" ", indentationSpacesBeforeContent) + + scanner := bufio.NewScanner(strings.NewReader(content)) + for scanner.Scan() { + line := scanner.Text() + indentedContent += (indentSpaces + line + "\n") + } + return indentedContent +} + +func getEndingLine(nodeIdx int, parentNode *yaml.Node) int { + node := parentNode.Content[nodeIdx] + if node.Kind == yaml.ScalarNode { + return node.Line + } + contentLen := len(node.Content) + + for idx := contentLen - 1; idx >= 0; idx-- { + if node.Content[idx].Line != 0 { + return getEndingLine(idx, node) + } + } + + return 0 +} + +func addFixesToFile(filePath string, lineAndContentsToAdd []LineAndContentToAdd) (cmdError error) { + linesSlice, err := getLinesSlice(filePath) + + if err != nil { + return err + } + + if err := os.Truncate(filePath, 0); err != nil { + return err + } + + file, err := os.OpenFile(filePath, os.O_RDWR, 0644) + if err != nil { + return err + } + + defer func() error { + if err := file.Close(); err != nil { + return err + } + return nil + }() + + writer := bufio.NewWriter(file) + lineIdx, lineToAddIdx := 0, 0 + + for lineToAddIdx < len(lineAndContentsToAdd) { + for lineIdx <= lineAndContentsToAdd[lineToAddIdx].Line { + _, err := writer.WriteString(linesSlice[lineIdx] + "\n") + if err != nil { + return err + } + lineIdx += 1 + } + + writeContentToAdd(writer, lineAndContentsToAdd[lineToAddIdx].Content) + lineToAddIdx += 1 + } + + for lineIdx < len(linesSlice) { + _, err := writer.WriteString(linesSlice[lineIdx] + "\n") + if err != nil { + return err + } + lineIdx += 1 + } + + writer.Flush() + return nil +} + +// Get the lines of existing yaml in a slice +func getLinesSlice(filePath string) ([]string, error) { + lineSlice := make([]string, 0) + + file, err := os.Open(filePath) + if err != nil { + log.Fatal(err) + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + lineSlice = append(lineSlice, scanner.Text()) + } + if err := scanner.Err(); err != nil { + log.Fatal(err) + return nil, err + } + + return lineSlice, err +} + +func writeContentToAdd(writer *bufio.Writer, contentToAdd string) { + scanner := bufio.NewScanner(strings.NewReader(contentToAdd)) + for scanner.Scan() { + line := scanner.Text() + writer.WriteString(line + "\n") + } +} From 2b6198907381e76fd8ad0f9f879ba863af4e56f4 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Tue, 29 Nov 2022 18:15:30 +0530 Subject: [PATCH 02/39] Fix local directory relative path --- core/pkg/resourcehandler/filesloader.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/pkg/resourcehandler/filesloader.go b/core/pkg/resourcehandler/filesloader.go index 933e392b..600a662c 100644 --- a/core/pkg/resourcehandler/filesloader.go +++ b/core/pkg/resourcehandler/filesloader.go @@ -103,6 +103,8 @@ func getResourcesFromPath(path string) (map[string]reporthandling.Source, []work gitRepo, err := cautils.NewLocalGitRepository(path) if err == nil && gitRepo != nil { repoRoot, _ = gitRepo.GetRootDir() + } else { + repoRoot, _ = filepath.Abs(path) } // load resource from local file system @@ -141,7 +143,7 @@ func getResourcesFromPath(path string) (map[string]reporthandling.Source, []work } workloadSource := reporthandling.Source{ - RelativePath: source, + RelativePath: relSource, FileType: filetype, LastCommit: lastCommit, } From 5ce69a750d738066ceda3538ed178860fee15e7b Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Tue, 29 Nov 2022 18:19:49 +0530 Subject: [PATCH 03/39] Fix git directory relative path --- core/cautils/scaninfo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/cautils/scaninfo.go b/core/cautils/scaninfo.go index 53ca9107..5f32bf04 100644 --- a/core/cautils/scaninfo.go +++ b/core/cautils/scaninfo.go @@ -419,7 +419,7 @@ func metadataGitLocal(input string) (*reporthandlingv2.RepoContextMetadata, erro Date: commit.Committer.Date, CommitterName: commit.Committer.Name, } - context.LocalRootPath = getAbsPath(input) + context.LocalRootPath, _ = gitParser.GetRootDir() return context, nil } From a7989bbe76101689254ba6728efbdcc9f1445082 Mon Sep 17 00:00:00 2001 From: Suhas Gumma <43647369+suhasgumma@users.noreply.github.com> Date: Wed, 30 Nov 2022 01:30:02 +0530 Subject: [PATCH 04/39] Update core/pkg/fixhandler/datastructures.go Co-authored-by: Vlad Klokun --- core/pkg/fixhandler/datastructures.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index a20fb09c..065f68b6 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -21,7 +21,7 @@ type ResourceFixInfo struct { FilePath string } -// LineAndContentToAdd is a struct that holds the information about where to insert the new changes in the existing yaml file +// LineAndContentToAdd holds the information about where to insert the new changes in the existing yaml file type LineAndContentToAdd struct { Line int Content string From f3665866afec896781dae6014b3353a1eaac0f6f Mon Sep 17 00:00:00 2001 From: Suhas Gumma <43647369+suhasgumma@users.noreply.github.com> Date: Wed, 30 Nov 2022 01:31:49 +0530 Subject: [PATCH 05/39] import "yqlib" properly Co-authored-by: Vlad Klokun --- core/pkg/fixhandler/yamlhandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index e83417e5..dafcc74a 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -10,7 +10,7 @@ import ( "log" "os" "strings" - "yqlib" + "github.com/mikefarah/yq/v4/pkg/yqlib" "gopkg.in/yaml.v3" ) From 4b898b0075a103ecf8135f6f05aee45015dd73e5 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Thu, 8 Dec 2022 22:56:53 +0530 Subject: [PATCH 06/39] Initial Implementation --- core/pkg/fixhandler/datastructures.go | 35 ++- core/pkg/fixhandler/fixhandler.go | 8 +- core/pkg/fixhandler/yamlhandler.go | 334 +++++++++++------------- core/pkg/fixhandler/yamlhelper.go | 276 ++++++++++++++++++++ examples/online-boutique/adservice.yaml | 17 +- 5 files changed, 463 insertions(+), 207 deletions(-) create mode 100644 core/pkg/fixhandler/yamlhelper.go diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index 065f68b6..50f69d2e 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -5,6 +5,7 @@ import ( metav1 "github.com/kubescape/kubescape/v2/core/meta/datastructures/v1" "github.com/kubescape/opa-utils/reporthandling" reporthandlingv2 "github.com/kubescape/opa-utils/reporthandling/v2" + "gopkg.in/yaml.v3" ) // FixHandler is a struct that holds the information of the report to be fixed @@ -21,8 +22,36 @@ type ResourceFixInfo struct { FilePath string } -// LineAndContentToAdd holds the information about where to insert the new changes in the existing yaml file -type LineAndContentToAdd struct { - Line int +// NodeInfo holds extra information about the node +type NodeInfo struct { + node *yaml.Node + parent *yaml.Node + + // position of the node among siblings + index int +} + +// FixInfoMetadata holds the arguments "getFixInfo" function needs to pass to the +// functions it uses +type FixInfoMetadata struct { + originalList *[]NodeInfo + fixedList *[]NodeInfo + originalListTracker int + fixedListTracker int + contentToAdd *[]ContentToAdd + contentToRemove *[]ContentToRemove +} + +// ContentToAdd holds the information about where to insert the new changes in the existing yaml file +type ContentToAdd struct { + // Line where the fix should be applied to + Line int + // Content is a string representation of the YAML node that describes a suggested fix Content string } + +// ContentToRemove holds the line numbers to remove from the existing yaml file +type ContentToRemove struct { + startLine int + endLine int +} diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index bfb6cc49..78f39bae 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -216,9 +216,13 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str } func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError error) { + originalYamlNode := getDecodedYaml(filePath) fixedYamlNode := getFixedYamlNode(filePath, yamlExpression) - lineAndContentsToAdd := getLineAndContentToAdd(&fixedYamlNode) - err := addFixesToFile(filePath, *lineAndContentsToAdd) + + originalList := getDFSOrder(originalYamlNode) + fixedList := getDFSOrder(fixedYamlNode) + contentToAdd, linesToRemove := getFixInfo(originalList, fixedList) + err := applyFixesToFile(filePath, contentToAdd, linesToRemove) return err } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index dafcc74a..5223e9a5 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -4,205 +4,187 @@ import ( "bufio" "bytes" "container/list" - "errors" "fmt" - "io" - "log" + "io/ioutil" "os" - "strings" + + logger "github.com/kubescape/go-logger" "github.com/mikefarah/yq/v4/pkg/yqlib" "gopkg.in/yaml.v3" ) -func getFixedYamlNode(filePath, yamlExpression string) yaml.Node { +func getDecodedYaml(filepath string) *yaml.Node { + file, err := ioutil.ReadFile(filepath) + if err != nil { + logger.L().Fatal("Cannot read file") + } + fileReader := bytes.NewReader(file) + dec := yaml.NewDecoder(fileReader) + + var node yaml.Node + err = dec.Decode(&node) + + if err != nil { + logger.L().Fatal("Cannot Decode Yaml") + } + + return &node +} + +func getFixedYamlNode(filePath, yamlExpression string) *yaml.Node { preferences := yqlib.ConfiguredYamlPreferences preferences.EvaluateTogether = true decoder := yqlib.NewYamlDecoder(preferences) var allDocuments = list.New() - reader, err := readStream(filePath) + reader, err := getNewReader(filePath) if err != nil { - return yaml.Node{} + return &yaml.Node{} } fileDocuments, err := readDocuments(reader, filePath, 0, decoder) if err != nil { - return yaml.Node{} + return &yaml.Node{} } allDocuments.PushBackList(fileDocuments) - if allDocuments.Len() == 0 { - candidateNode := &yqlib.CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, - FileIndex: 0, - LeadingContent: "", - } - allDocuments.PushBack(candidateNode) - } - allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator() - matches, _ := allAtOnceEvaluator.EvaluateCandidateNodes(yamlExpression, allDocuments) + matches, err := allAtOnceEvaluator.EvaluateCandidateNodes(yamlExpression, allDocuments) - return *matches.Front().Value.(*yqlib.CandidateNode).Node -} - -func readStream(filename string) (io.Reader, error) { - var reader *bufio.Reader - if filename == "-" { - reader = bufio.NewReader(os.Stdin) - } else { - // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, - // and ensuring that it's not possible to give a path to a file outside thar directory. - file, err := os.Open(filename) // #nosec - if err != nil { - return nil, err - } - reader = bufio.NewReader(file) - } - return reader, nil - -} - -func readDocuments(reader io.Reader, filename string, fileIndex int, decoder yqlib.Decoder) (*list.List, error) { - err := decoder.Init(reader) if err != nil { - return nil, err + logger.L().Fatal(fmt.Sprintf("Error fixing YAML, %v", err.Error())) } - inputList := list.New() - var currentIndex uint - for { - candidateNode, errorReading := decoder.Decode() + return matches.Front().Value.(*yqlib.CandidateNode).Node +} - if errors.Is(errorReading, io.EOF) { - switch reader := reader.(type) { - case *os.File: - safelyCloseFile(reader) - } - return inputList, nil - } else if errorReading != nil { - return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading) - } - candidateNode.Document = currentIndex - candidateNode.Filename = filename - candidateNode.FileIndex = fileIndex - candidateNode.EvaluateTogether = true +func getDFSOrder(node *yaml.Node) *[]NodeInfo { + dfsOrder := make([]NodeInfo, 0) + getDFSOrderHelper(node, nil, &dfsOrder, 0) + return &dfsOrder +} - inputList.PushBack(candidateNode) +func matchNodes(nodeOne, nodeTwo *yaml.Node) int { - currentIndex = currentIndex + 1 + isNewNode := nodeTwo.Line == 0 && nodeTwo.Column == 0 + sameLines := nodeOne.Line == nodeTwo.Line + sameColumns := nodeOne.Column == nodeTwo.Column + sameKinds := nodeOne.Kind == nodeTwo.Kind + sameValues := nodeOne.Value == nodeTwo.Value + + isSameNode := sameKinds && sameValues && sameLines && sameColumns + + switch { + case isSameNode: + return int(sameNodes) + case isNewNode: + return int(insertedNode) + case sameLines && sameColumns: + return int(replacedNode) + default: + return int(removedNode) } } -func safelyCloseFile(file *os.File) { - err := file.Close() - if err != nil { - fmt.Println("Error Closing File") +func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]ContentToRemove) { + contentToAdd := make([]ContentToAdd, 0) + linesToRemove := make([]ContentToRemove, 0) + + originalListTracker, fixedListTracker := 0, 0 + + fixInfoMetadata := &FixInfoMetadata{ + originalList: originalList, + fixedList: fixedList, + originalListTracker: originalListTracker, + fixedListTracker: fixedListTracker, + contentToAdd: &contentToAdd, + contentToRemove: &linesToRemove, } + + for originalListTracker < len(*originalList) && fixedListTracker < len(*fixedList) { + matchNodeResult := matchNodes((*originalList)[originalListTracker].node, (*fixedList)[fixedListTracker].node) + + fixInfoMetadata.originalListTracker = originalListTracker + fixInfoMetadata.fixedListTracker = fixedListTracker + + switch matchNodeResult { + case int(sameNodes): + originalListTracker += 1 + fixedListTracker += 1 + + case int(removedNode): + originalListTracker = addLinesToRemove(fixInfoMetadata) + + case int(insertedNode): + fixedListTracker = addLinesToInsert(fixInfoMetadata) + + case int(replacedNode): + originalListTracker, fixedListTracker = updateLinesToReplace(fixInfoMetadata) + } + } + + for originalListTracker < len(*originalList) { + fixInfoMetadata.originalListTracker = originalListTracker + fixInfoMetadata.fixedListTracker = len(*fixedList) - 1 + originalListTracker = addLinesToRemove(fixInfoMetadata) + } + + for fixedListTracker < len(*fixedList) { + fixInfoMetadata.originalListTracker = len(*originalList) - 1 + fixInfoMetadata.fixedListTracker = fixedListTracker + fixedListTracker = addLinesToInsert(fixInfoMetadata) + } + + return &contentToAdd, &linesToRemove + } -func getLineAndContentToAdd(node *yaml.Node) *[]LineAndContentToAdd { - contentToAdd := make([]LineAndContentToAdd, 0) - getLineAndContentToAddHelper(0, node, &contentToAdd) - return &contentToAdd +// Adds the lines to remove and returns the updated originalListTracker +func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) int { + currentDFSNode := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker] + newTracker := updateTracker(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) + *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ + startLine: currentDFSNode.node.Line, + endLine: getNodeLine(fixInfoMetadata.originalList, newTracker) - 1, + }) + + return newTracker } -func getLineAndContentToAddHelper(nodeIdx int, parentNode *yaml.Node, contentToAdd *[]LineAndContentToAdd) { - node := parentNode.Content[nodeIdx] - var content string - var err error - if node.Line == 0 && node.Column == 0 { - if parentNode.Kind == yaml.MappingNode { - if nodeIdx%2 != 0 { - return - } - } - content, err = enocodeIntoYaml(parentNode, nodeIdx) +// Adds the lines to insert and returns the updated fixedListTracker +func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) int { + currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] + lineToInsert := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker].node.Line - 1 + contentToInsert := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) - if err != nil { - fmt.Println("Cannot Encode into YAML") - } + newTracker := updateTracker(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) - indentationSpacesBeforeContent := parentNode.Column - 1 + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ + Line: lineToInsert, + Content: contentToInsert, + }) - content = addIndentationToContent(content, indentationSpacesBeforeContent) - - // Getting the line to add content after. Add directly after the left Sibling. - var lineToAddAfter int - for idx := nodeIdx - 1; idx >= 0; idx-- { - if parentNode.Content[idx].Line != 0 { - lineToAddAfter = getEndingLine(idx, parentNode) - } - } - lineAndContentToAdd := LineAndContentToAdd{ - Line: lineToAddAfter, - Content: content, - } - *contentToAdd = append(*contentToAdd, lineAndContentToAdd) - } - - for index, _ := range node.Content { - getLineAndContentToAddHelper(index, node, contentToAdd) - } + return newTracker } -func enocodeIntoYaml(parentNode *yaml.Node, nodeIdx int) (string, error) { - if parentNode.Kind == yaml.MappingNode { - content := make([]*yaml.Node, 0) - content = append(content, parentNode.Content[nodeIdx], parentNode.Content[nodeIdx+1]) - parentForContent := yaml.Node{ - Kind: yaml.MappingNode, - Content: content, - } - buf := new(bytes.Buffer) - encoder := yaml.NewEncoder(buf) - errorEncoding := encoder.Encode(parentForContent) - if errorEncoding != nil { - return "", fmt.Errorf("Error debugging node, %v", errorEncoding.Error()) - } - errorClosingEncoder := encoder.Close() - if errorClosingEncoder != nil { - return "", fmt.Errorf("Error closing encoder: ", errorClosingEncoder.Error()) - } - return fmt.Sprintf(`%v`, buf.String()), nil +// Adds the lines to remove and insert and updates the fixedListTracker and originalListTracker +func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { + currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] + + if isValueNodeinMapping(¤tDFSNode) { + fixInfoMetadata.originalListTracker -= 1 + fixInfoMetadata.fixedListTracker -= 1 } - return "", nil + updatedOriginalTracker := addLinesToRemove(fixInfoMetadata) + updatedFixedTracker := addLinesToInsert(fixInfoMetadata) + + return updatedOriginalTracker, updatedFixedTracker } -func addIndentationToContent(content string, indentationSpacesBeforeContent int) string { - indentedContent := "" - indentSpaces := strings.Repeat(" ", indentationSpacesBeforeContent) - - scanner := bufio.NewScanner(strings.NewReader(content)) - for scanner.Scan() { - line := scanner.Text() - indentedContent += (indentSpaces + line + "\n") - } - return indentedContent -} - -func getEndingLine(nodeIdx int, parentNode *yaml.Node) int { - node := parentNode.Content[nodeIdx] - if node.Kind == yaml.ScalarNode { - return node.Line - } - contentLen := len(node.Content) - - for idx := contentLen - 1; idx >= 0; idx-- { - if node.Content[idx].Line != 0 { - return getEndingLine(idx, node) - } - } - - return 0 -} - -func addFixesToFile(filePath string, lineAndContentsToAdd []LineAndContentToAdd) (cmdError error) { +func applyFixesToFile(filePath string, lineAndContentsToAdd *[]ContentToAdd, linesToRemove *[]ContentToRemove) (cmdError error) { linesSlice, err := getLinesSlice(filePath) if err != nil { @@ -225,11 +207,16 @@ func addFixesToFile(filePath string, lineAndContentsToAdd []LineAndContentToAdd) return nil }() + removeLines(linesToRemove, &linesSlice) + writer := bufio.NewWriter(file) lineIdx, lineToAddIdx := 0, 0 - for lineToAddIdx < len(lineAndContentsToAdd) { - for lineIdx <= lineAndContentsToAdd[lineToAddIdx].Line { + for lineToAddIdx < len(*lineAndContentsToAdd) { + for lineIdx <= (*lineAndContentsToAdd)[lineToAddIdx].Line { + if linesSlice[lineIdx] == "*" { + continue + } _, err := writer.WriteString(linesSlice[lineIdx] + "\n") if err != nil { return err @@ -237,11 +224,14 @@ func addFixesToFile(filePath string, lineAndContentsToAdd []LineAndContentToAdd) lineIdx += 1 } - writeContentToAdd(writer, lineAndContentsToAdd[lineToAddIdx].Content) + writeContentToAdd(writer, (*lineAndContentsToAdd)[lineToAddIdx].Content) lineToAddIdx += 1 } for lineIdx < len(linesSlice) { + if linesSlice[lineIdx] == "*" { + continue + } _, err := writer.WriteString(linesSlice[lineIdx] + "\n") if err != nil { return err @@ -252,35 +242,3 @@ func addFixesToFile(filePath string, lineAndContentsToAdd []LineAndContentToAdd) writer.Flush() return nil } - -// Get the lines of existing yaml in a slice -func getLinesSlice(filePath string) ([]string, error) { - lineSlice := make([]string, 0) - - file, err := os.Open(filePath) - if err != nil { - log.Fatal(err) - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - - for scanner.Scan() { - lineSlice = append(lineSlice, scanner.Text()) - } - if err := scanner.Err(); err != nil { - log.Fatal(err) - return nil, err - } - - return lineSlice, err -} - -func writeContentToAdd(writer *bufio.Writer, contentToAdd string) { - scanner := bufio.NewScanner(strings.NewReader(contentToAdd)) - for scanner.Scan() { - line := scanner.Text() - writer.WriteString(line + "\n") - } -} diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go new file mode 100644 index 00000000..ae2cc5ae --- /dev/null +++ b/core/pkg/fixhandler/yamlhelper.go @@ -0,0 +1,276 @@ +package fixhandler + +import ( + "bufio" + "bytes" + "container/list" + "errors" + "fmt" + "io" + "log" + "math" + "os" + "strings" + + logger "github.com/kubescape/go-logger" + "github.com/mikefarah/yq/v4/pkg/yqlib" + "gopkg.in/yaml.v3" +) + +const ( + sameNodes = iota + insertedNode + removedNode + replacedNode +) + +func getNewReader(filename string) (io.Reader, error) { + var reader *bufio.Reader + if filename == "-" { + reader = bufio.NewReader(os.Stdin) + } else { + // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, + // and ensuring that it's not possible to give a path to a file outside thar directory. + file, err := os.Open(filename) // #nosec + if err != nil { + return nil, err + } + reader = bufio.NewReader(file) + } + return reader, nil +} + +func readDocuments(reader io.Reader, filename string, fileIndex int, decoder yqlib.Decoder) (*list.List, error) { + err := decoder.Init(reader) + if err != nil { + return nil, err + } + inputList := list.New() + var currentIndex uint + + for { + candidateNode, errorReading := decoder.Decode() + + if errors.Is(errorReading, io.EOF) { + switch reader := reader.(type) { + case *os.File: + safelyCloseFile(reader) + } + return inputList, nil + } else if errorReading != nil { + return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading) + } + candidateNode.Document = currentIndex + candidateNode.Filename = filename + candidateNode.FileIndex = fileIndex + candidateNode.EvaluateTogether = true + + inputList.PushBack(candidateNode) + + currentIndex = currentIndex + 1 + } +} + +func safelyCloseFile(file *os.File) { + err := file.Close() + if err != nil { + logger.L().Error("Error Closing File") + } +} + +func getDFSOrderHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]NodeInfo, index int) { + dfsNode := NodeInfo{ + node: node, + parent: parent, + index: index, + } + + fmt.Println(dfsNode.node) + fmt.Println(dfsNode.parent) + fmt.Println(dfsNode.index) + + *dfsOrder = append(*dfsOrder, dfsNode) + + for idx, child := range node.Content { + getDFSOrderHelper(child, node, dfsOrder, idx) + } +} + +// Skips the current node including it's children in DFS order and returns the new tracker. +func skipCurrentNode(node *yaml.Node, currentTracker int) int { + updatedTracker := currentTracker + getChildrenCount(node) + return updatedTracker +} + +func getChildrenCount(node *yaml.Node) int { + totalChildren := 1 + for _, child := range node.Content { + totalChildren += getChildrenCount(child) + } + return totalChildren +} + +// Moves the tracker to the parent of given node +func traceBackToParent(dfsOrder *[]NodeInfo, currentTracker int) int { + parentNode := (*dfsOrder)[currentTracker].parent + parentIdx := currentTracker - 1 + for parentIdx >= 0 { + if (*dfsOrder)[parentIdx].node == parentNode { + return parentIdx + } + parentIdx -= 1 + } + return 0 +} + +// Checks if the node is value node in "key-value" pairs of mapping node +func isValueNodeinMapping(dfsNode *NodeInfo) bool { + if dfsNode.parent.Kind == yaml.MappingNode && dfsNode.index%2 != 0 { + return true + } + return false +} + +func updateTracker(dfsOrder *[]NodeInfo, tracker int) int { + currentDFSNode := (*dfsOrder)[tracker] + var newTracker int + + if currentDFSNode.parent.Kind == yaml.MappingNode { + valueNode := (*dfsOrder)[tracker+1] + newTracker = skipCurrentNode(valueNode.node, tracker+1) + } else { + newTracker = skipCurrentNode(currentDFSNode.node, tracker) + } + + return newTracker +} + +func getNodeLine(dfsOrder *[]NodeInfo, tracker int) int { + if tracker < len(*dfsOrder) { + return (*dfsOrder)[tracker].node.Line + } else { + return int(math.Inf(1)) + } +} + +func isOneLineSequenceNode(node *yaml.Node) bool { + if node.Kind != yaml.SequenceNode { + return false + } + nodeLine := node.Line + for _, child := range node.Content { + if child.Line != nodeLine { + return false + } + } + return true +} + +func enocodeIntoYaml(parentNode *yaml.Node, dfsOrder *[]NodeInfo, tracker int) (string, error) { + content := make([]*yaml.Node, 0) + currentNode := (*dfsOrder)[tracker].node + content = append(content, currentNode) + + if parentNode.Kind == yaml.MappingNode { + valueNode := (*dfsOrder)[tracker+1].node + content = append(content, valueNode) + } + + parentForContent := yaml.Node{ + Kind: parentNode.Kind, + Content: content, + } + buf := new(bytes.Buffer) + encoder := yaml.NewEncoder(buf) + errorEncoding := encoder.Encode(parentForContent) + if errorEncoding != nil { + return "", fmt.Errorf("Error debugging node, %v", errorEncoding.Error()) + } + errorClosingEncoder := encoder.Close() + if errorClosingEncoder != nil { + return "", fmt.Errorf("Error closing encoder: %v", errorClosingEncoder.Error()) + } + return fmt.Sprintf(`%v`, buf.String()), nil +} + +func getContent(parentNode *yaml.Node, dfsOrder *[]NodeInfo, tracker int) string { + content, err := enocodeIntoYaml(parentNode, dfsOrder, tracker) + if err != nil { + logger.L().Fatal("Cannot Encode into YAML") + } + + indentationSpaces := parentNode.Column - 1 + + content = indentContent(content, indentationSpaces) + + return content +} + +func indentContent(content string, indentationSpaces int) string { + indentedContent := "" + indentSpaces := strings.Repeat(" ", indentationSpaces) + + scanner := bufio.NewScanner(strings.NewReader(content)) + for scanner.Scan() { + line := scanner.Text() + indentedContent += (indentSpaces + line + "\n") + } + return indentedContent +} + +func removeLines(linesToRemove *[]ContentToRemove, linesSlice *[]string) { + for _, lineToRemove := range *linesToRemove { + startLine := lineToRemove.startLine + endLine := int(math.Min(float64(lineToRemove.endLine), float64(len(*linesSlice)-1))) + + for line := startLine; line <= endLine; line++ { + lineContent := strings.ReplaceAll((*linesSlice)[line], " ", "") + if isEmptyLineOrComment(lineContent) { + break + } + (*linesSlice)[line] = "*" + } + } +} + +// Checks if the line is empty or a comment +func isEmptyLineOrComment(lineContent string) bool { + if lineContent == "" { + return true + } else if lineContent[0:1] == "#" { + return true + } + return false +} + +// Get the lines of existing yaml in a slice +func getLinesSlice(filePath string) ([]string, error) { + lineSlice := make([]string, 0) + + file, err := os.Open(filePath) + if err != nil { + log.Fatal(err) + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + lineSlice = append(lineSlice, scanner.Text()) + } + if err := scanner.Err(); err != nil { + log.Fatal(err) + return nil, err + } + + return lineSlice, err +} + +func writeContentToAdd(writer *bufio.Writer, contentToAdd string) { + scanner := bufio.NewScanner(strings.NewReader(contentToAdd)) + for scanner.Scan() { + line := scanner.Text() + writer.WriteString(line + "\n") + } +} diff --git a/examples/online-boutique/adservice.yaml b/examples/online-boutique/adservice.yaml index 78a234dd..fd9f4831 100644 --- a/examples/online-boutique/adservice.yaml +++ b/examples/online-boutique/adservice.yaml @@ -1,17 +1,3 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - apiVersion: apps/v1 kind: Deployment metadata: @@ -58,6 +44,9 @@ spec: periodSeconds: 15 exec: command: ["/bin/grpc_health_probe", "-addr=:9555"] + securityContext: + allowPrivilegeEscalation: false + --- apiVersion: v1 kind: Service From 7fcfa27d9a787039c9e0fbc33f4c164fba0a2793 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Thu, 8 Dec 2022 23:06:05 +0530 Subject: [PATCH 07/39] Initial Implementation --- examples/online-boutique/adservice.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/examples/online-boutique/adservice.yaml b/examples/online-boutique/adservice.yaml index fd9f4831..78a234dd 100644 --- a/examples/online-boutique/adservice.yaml +++ b/examples/online-boutique/adservice.yaml @@ -1,3 +1,17 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + apiVersion: apps/v1 kind: Deployment metadata: @@ -44,9 +58,6 @@ spec: periodSeconds: 15 exec: command: ["/bin/grpc_health_probe", "-addr=:9555"] - securityContext: - allowPrivilegeEscalation: false - --- apiVersion: v1 kind: Service From 30e5b9b57db5e8b4c58e422777aa9d319b74e838 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Fri, 9 Dec 2022 02:29:14 +0530 Subject: [PATCH 08/39] Insert Scenarios --- core/pkg/fixhandler/yamlhelper.go | 5 -- .../fixed_yaml_scenario_1.yml | 14 ++++ .../fixed_yaml_scenario_2.yml | 15 ++++ .../fixed_yaml_scenario_3.yml | 16 +++++ .../fixed_yaml_scenario_4.yml | 70 +++++++++++++++++++ .../fixed_yaml_scenario_5.yml | 18 +++++ .../fixed_yaml_scenario_6.yml | 16 +++++ .../original_yaml_scenario_1.yml | 12 ++++ .../original_yaml_scenario_2.yml | 11 +++ .../original_yaml_scenario_3.yml | 15 ++++ .../original_yaml_scenario_4.yml | 60 ++++++++++++++++ .../original_yaml_scenario_5.yml | 16 +++++ .../original_yaml_scenario_6.yml | 14 ++++ 13 files changed, 277 insertions(+), 5 deletions(-) create mode 100644 examples/fix_command/insert_scenarios/fixed_yaml_scenario_1.yml create mode 100644 examples/fix_command/insert_scenarios/fixed_yaml_scenario_2.yml create mode 100644 examples/fix_command/insert_scenarios/fixed_yaml_scenario_3.yml create mode 100644 examples/fix_command/insert_scenarios/fixed_yaml_scenario_4.yml create mode 100644 examples/fix_command/insert_scenarios/fixed_yaml_scenario_5.yml create mode 100644 examples/fix_command/insert_scenarios/fixed_yaml_scenario_6.yml create mode 100644 examples/fix_command/insert_scenarios/original_yaml_scenario_1.yml create mode 100644 examples/fix_command/insert_scenarios/original_yaml_scenario_2.yml create mode 100644 examples/fix_command/insert_scenarios/original_yaml_scenario_3.yml create mode 100644 examples/fix_command/insert_scenarios/original_yaml_scenario_4.yml create mode 100644 examples/fix_command/insert_scenarios/original_yaml_scenario_5.yml create mode 100644 examples/fix_command/insert_scenarios/original_yaml_scenario_6.yml diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index ae2cc5ae..f023a959 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -84,11 +84,6 @@ func getDFSOrderHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]NodeInfo, parent: parent, index: index, } - - fmt.Println(dfsNode.node) - fmt.Println(dfsNode.parent) - fmt.Println(dfsNode.index) - *dfsOrder = append(*dfsOrder, dfsNode) for idx, child := range node.Content { diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_1.yml b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_1.yml new file mode 100644 index 00000000..55f3aeb5 --- /dev/null +++ b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_1.yml @@ -0,0 +1,14 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + allowPrivilegeEscalation: false \ No newline at end of file diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_2.yml b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_2.yml new file mode 100644 index 00000000..6ffcf01a --- /dev/null +++ b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_2.yml @@ -0,0 +1,15 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: + - NET_RAW \ No newline at end of file diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_3.yml b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_3.yml new file mode 100644 index 00000000..1dfcfd49 --- /dev/null +++ b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_3.yml @@ -0,0 +1,16 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["SYS_ADM"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: + - NET_RAW + - SYS_ADM \ No newline at end of file diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_4.yml b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_4.yml new file mode 100644 index 00000000..ebe4c8fc --- /dev/null +++ b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_4.yml @@ -0,0 +1,70 @@ +# Fixes to Apply: +# 1) spec.template.spec.securityContext.allowPrivilegeEscalation = false +# 2) spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] +# 3) spec.template.spec.containers[0].securityContext.seccompProfile.type = RuntimeDefault +# 4) spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation = false +# 5) spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem = true + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: multiple_inserts +spec: + selector: + matchLabels: + app: example_4 + template: + metadata: + labels: + app: example_4 + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + containers: + - name: example_4 + image: nginx + ports: + - containerPort: 3000 + env: + - name: PORT + value: "3000" + resources: + requests: + cpu: 200m + memory: 180Mi + limits: + cpu: 300m + memory: 300Mi + readinessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:3000"] + livenessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:3000"] + securityContext: + capabilities: + drop: + - "NET_RAW" + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + securityContext: + allowPrivilegeEscalation: false +--- +apiVersion: v1 +kind: Service +metadata: + name: example_4 +spec: + type: ClusterIP + selector: + app: example_4 + ports: + - name: test_port + port: 3000 + targetPort: 3000 diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_5.yml b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_5.yml new file mode 100644 index 00000000..c9917375 --- /dev/null +++ b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_5.yml @@ -0,0 +1,18 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + allowPrivilegeEscalation: false + + # Testing if comments are retained as intended + securityContext: + runAsRoot: false \ No newline at end of file diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_6.yml b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_6.yml new file mode 100644 index 00000000..6eea6f7f --- /dev/null +++ b/examples/fix_command/insert_scenarios/fixed_yaml_scenario_6.yml @@ -0,0 +1,16 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["SYS_ADM"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: + - "NET_RAW" + - "SYS_ADM" \ No newline at end of file diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_1.yml b/examples/fix_command/insert_scenarios/original_yaml_scenario_1.yml new file mode 100644 index 00000000..7c15eec3 --- /dev/null +++ b/examples/fix_command/insert_scenarios/original_yaml_scenario_1.yml @@ -0,0 +1,12 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx \ No newline at end of file diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_2.yml b/examples/fix_command/insert_scenarios/original_yaml_scenario_2.yml new file mode 100644 index 00000000..d88937a4 --- /dev/null +++ b/examples/fix_command/insert_scenarios/original_yaml_scenario_2.yml @@ -0,0 +1,11 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx_container + image: nginx \ No newline at end of file diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_3.yml b/examples/fix_command/insert_scenarios/original_yaml_scenario_3.yml new file mode 100644 index 00000000..ec8f58e9 --- /dev/null +++ b/examples/fix_command/insert_scenarios/original_yaml_scenario_3.yml @@ -0,0 +1,15 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["SYS_ADM"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: + - NET_RAW \ No newline at end of file diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_4.yml b/examples/fix_command/insert_scenarios/original_yaml_scenario_4.yml new file mode 100644 index 00000000..4a4b896a --- /dev/null +++ b/examples/fix_command/insert_scenarios/original_yaml_scenario_4.yml @@ -0,0 +1,60 @@ +# Fixes to Apply: +# 1) spec.template.spec.securityContext.allowPrivilegeEscalation = false +# 2) spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] +# 3) spec.template.spec.containers[0].securityContext.seccompProfile.type = RuntimeDefault +# 4) spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation = false +# 5) spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem = true + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: multiple_inserts +spec: + selector: + matchLabels: + app: example_4 + template: + metadata: + labels: + app: example_4 + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + containers: + - name: example_4 + image: nginx + ports: + - containerPort: 3000 + env: + - name: PORT + value: "3000" + resources: + requests: + cpu: 200m + memory: 180Mi + limits: + cpu: 300m + memory: 300Mi + readinessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:3000"] + livenessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:3000"] +--- +apiVersion: v1 +kind: Service +metadata: + name: example_4 +spec: + type: ClusterIP + selector: + app: example_4 + ports: + - name: test_port + port: 3000 + targetPort: 3000 diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_5.yml b/examples/fix_command/insert_scenarios/original_yaml_scenario_5.yml new file mode 100644 index 00000000..7e62d9f8 --- /dev/null +++ b/examples/fix_command/insert_scenarios/original_yaml_scenario_5.yml @@ -0,0 +1,16 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + + # Testing if comments are retained as intended + securityContext: + runAsRoot: false \ No newline at end of file diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_6.yml b/examples/fix_command/insert_scenarios/original_yaml_scenario_6.yml new file mode 100644 index 00000000..383e5a45 --- /dev/null +++ b/examples/fix_command/insert_scenarios/original_yaml_scenario_6.yml @@ -0,0 +1,14 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["SYS_ADM"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: ["NET_RAW"] \ No newline at end of file From 252a564552123051efb1c292f75ef812cc7ea617 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Fri, 9 Dec 2022 12:23:51 +0530 Subject: [PATCH 09/39] Remove, Replace and Hybrid Scenarios --- .../fixed_yaml_scenario_1.yml | 19 +++++++++++++++++++ .../original_yaml_scenario_1.yml | 19 +++++++++++++++++++ .../fixed_yaml_scenario_1.yml | 12 ++++++++++++ .../fixed_yaml_scenario_2.yml | 12 ++++++++++++ .../fixed_yaml_scenario_3.yml | 15 +++++++++++++++ .../original_yaml_scenario_1.yml | 14 ++++++++++++++ .../original_yaml_scenario_2.yml | 15 +++++++++++++++ .../original_yaml_scenario_3.yml | 14 ++++++++++++++ .../fixed_yaml_scenario_1.yml | 14 ++++++++++++++ .../fixed_yaml_scenario_2.yml | 19 +++++++++++++++++++ .../original_yaml_scenario_1.yml | 14 ++++++++++++++ .../original_yaml_scenario_2.yml | 18 ++++++++++++++++++ 12 files changed, 185 insertions(+) create mode 100644 examples/fix_command/hybrid_scenarios/fixed_yaml_scenario_1.yml create mode 100644 examples/fix_command/hybrid_scenarios/original_yaml_scenario_1.yml create mode 100644 examples/fix_command/remove_scenarios/fixed_yaml_scenario_1.yml create mode 100644 examples/fix_command/remove_scenarios/fixed_yaml_scenario_2.yml create mode 100644 examples/fix_command/remove_scenarios/fixed_yaml_scenario_3.yml create mode 100644 examples/fix_command/remove_scenarios/original_yaml_scenario_1.yml create mode 100644 examples/fix_command/remove_scenarios/original_yaml_scenario_2.yml create mode 100644 examples/fix_command/remove_scenarios/original_yaml_scenario_3.yml create mode 100644 examples/fix_command/replace_scenarios/fixed_yaml_scenario_1.yml create mode 100644 examples/fix_command/replace_scenarios/fixed_yaml_scenario_2.yml create mode 100644 examples/fix_command/replace_scenarios/original_yaml_scenario_1.yml create mode 100644 examples/fix_command/replace_scenarios/original_yaml_scenario_2.yml diff --git a/examples/fix_command/hybrid_scenarios/fixed_yaml_scenario_1.yml b/examples/fix_command/hybrid_scenarios/fixed_yaml_scenario_1.yml new file mode 100644 index 00000000..5ab9f08c --- /dev/null +++ b/examples/fix_command/hybrid_scenarios/fixed_yaml_scenario_1.yml @@ -0,0 +1,19 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false \ No newline at end of file diff --git a/examples/fix_command/hybrid_scenarios/original_yaml_scenario_1.yml b/examples/fix_command/hybrid_scenarios/original_yaml_scenario_1.yml new file mode 100644 index 00000000..c2afc6bd --- /dev/null +++ b/examples/fix_command/hybrid_scenarios/original_yaml_scenario_1.yml @@ -0,0 +1,19 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: true \ No newline at end of file diff --git a/examples/fix_command/remove_scenarios/fixed_yaml_scenario_1.yml b/examples/fix_command/remove_scenarios/fixed_yaml_scenario_1.yml new file mode 100644 index 00000000..8ecc8724 --- /dev/null +++ b/examples/fix_command/remove_scenarios/fixed_yaml_scenario_1.yml @@ -0,0 +1,12 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx \ No newline at end of file diff --git a/examples/fix_command/remove_scenarios/fixed_yaml_scenario_2.yml b/examples/fix_command/remove_scenarios/fixed_yaml_scenario_2.yml new file mode 100644 index 00000000..55cf481a --- /dev/null +++ b/examples/fix_command/remove_scenarios/fixed_yaml_scenario_2.yml @@ -0,0 +1,12 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx \ No newline at end of file diff --git a/examples/fix_command/remove_scenarios/fixed_yaml_scenario_3.yml b/examples/fix_command/remove_scenarios/fixed_yaml_scenario_3.yml new file mode 100644 index 00000000..254affe4 --- /dev/null +++ b/examples/fix_command/remove_scenarios/fixed_yaml_scenario_3.yml @@ -0,0 +1,15 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: + - NET_RAW \ No newline at end of file diff --git a/examples/fix_command/remove_scenarios/original_yaml_scenario_1.yml b/examples/fix_command/remove_scenarios/original_yaml_scenario_1.yml new file mode 100644 index 00000000..579ce100 --- /dev/null +++ b/examples/fix_command/remove_scenarios/original_yaml_scenario_1.yml @@ -0,0 +1,14 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false \ No newline at end of file diff --git a/examples/fix_command/remove_scenarios/original_yaml_scenario_2.yml b/examples/fix_command/remove_scenarios/original_yaml_scenario_2.yml new file mode 100644 index 00000000..4b1f39d5 --- /dev/null +++ b/examples/fix_command/remove_scenarios/original_yaml_scenario_2.yml @@ -0,0 +1,15 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + + - name: container_with_security_issues + image: image_with_security_issues \ No newline at end of file diff --git a/examples/fix_command/remove_scenarios/original_yaml_scenario_3.yml b/examples/fix_command/remove_scenarios/original_yaml_scenario_3.yml new file mode 100644 index 00000000..c53829a8 --- /dev/null +++ b/examples/fix_command/remove_scenarios/original_yaml_scenario_3.yml @@ -0,0 +1,14 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: ["NET_RAW", "SYS_ADM"] \ No newline at end of file diff --git a/examples/fix_command/replace_scenarios/fixed_yaml_scenario_1.yml b/examples/fix_command/replace_scenarios/fixed_yaml_scenario_1.yml new file mode 100644 index 00000000..4a4eda33 --- /dev/null +++ b/examples/fix_command/replace_scenarios/fixed_yaml_scenario_1.yml @@ -0,0 +1,14 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.runAsRoot |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false \ No newline at end of file diff --git a/examples/fix_command/replace_scenarios/fixed_yaml_scenario_2.yml b/examples/fix_command/replace_scenarios/fixed_yaml_scenario_2.yml new file mode 100644 index 00000000..89775e04 --- /dev/null +++ b/examples/fix_command/replace_scenarios/fixed_yaml_scenario_2.yml @@ -0,0 +1,19 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" +# select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW" + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: + - "SYS_ADM" + add: + - "NET_RAW" \ No newline at end of file diff --git a/examples/fix_command/replace_scenarios/original_yaml_scenario_1.yml b/examples/fix_command/replace_scenarios/original_yaml_scenario_1.yml new file mode 100644 index 00000000..335585b2 --- /dev/null +++ b/examples/fix_command/replace_scenarios/original_yaml_scenario_1.yml @@ -0,0 +1,14 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.runAsRoot |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: true \ No newline at end of file diff --git a/examples/fix_command/replace_scenarios/original_yaml_scenario_2.yml b/examples/fix_command/replace_scenarios/original_yaml_scenario_2.yml new file mode 100644 index 00000000..4a4821c0 --- /dev/null +++ b/examples/fix_command/replace_scenarios/original_yaml_scenario_2.yml @@ -0,0 +1,18 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" +# select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW" + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: + - "NET_RAW" + add: ["SYS_ADM"] \ No newline at end of file From 48516b891f27a71a225b3d9d88e474155388054b Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Fri, 9 Dec 2022 15:31:09 +0530 Subject: [PATCH 10/39] unit tests --- core/pkg/fixhandler/fixhandler_test.go | 142 ++++++++++++++---- .../fixed_yaml_scenario_1.yml | 0 .../original_yaml_scenario_1.yml | 0 .../fixed_yaml_scenario_1.yml | 0 .../fixed_yaml_scenario_2.yml | 0 .../fixed_yaml_scenario_3.yml | 0 .../fixed_yaml_scenario_4.yml | 10 +- .../fixed_yaml_scenario_5.yml | 0 .../fixed_yaml_scenario_6.yml | 0 .../original_yaml_scenario_1.yml | 0 .../original_yaml_scenario_2.yml | 0 .../original_yaml_scenario_3.yml | 0 .../original_yaml_scenario_4.yml | 10 +- .../original_yaml_scenario_5.yml | 0 .../original_yaml_scenario_6.yml | 0 .../fixed_yaml_scenario_1.yml | 0 .../fixed_yaml_scenario_2.yml | 0 .../fixed_yaml_scenario_3.yml | 0 .../original_yaml_scenario_1.yml | 0 .../original_yaml_scenario_2.yml | 0 .../original_yaml_scenario_3.yml | 0 .../fixed_yaml_scenario_1.yml | 0 .../fixed_yaml_scenario_2.yml | 0 .../original_yaml_scenario_1.yml | 0 .../original_yaml_scenario_2.yml | 0 25 files changed, 121 insertions(+), 41 deletions(-) rename examples/{fix_command => fix-command}/hybrid_scenarios/fixed_yaml_scenario_1.yml (100%) rename examples/{fix_command => fix-command}/hybrid_scenarios/original_yaml_scenario_1.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/fixed_yaml_scenario_1.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/fixed_yaml_scenario_2.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/fixed_yaml_scenario_3.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/fixed_yaml_scenario_4.yml (75%) rename examples/{fix_command => fix-command}/insert_scenarios/fixed_yaml_scenario_5.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/fixed_yaml_scenario_6.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/original_yaml_scenario_1.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/original_yaml_scenario_2.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/original_yaml_scenario_3.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/original_yaml_scenario_4.yml (70%) rename examples/{fix_command => fix-command}/insert_scenarios/original_yaml_scenario_5.yml (100%) rename examples/{fix_command => fix-command}/insert_scenarios/original_yaml_scenario_6.yml (100%) rename examples/{fix_command => fix-command}/remove_scenarios/fixed_yaml_scenario_1.yml (100%) rename examples/{fix_command => fix-command}/remove_scenarios/fixed_yaml_scenario_2.yml (100%) rename examples/{fix_command => fix-command}/remove_scenarios/fixed_yaml_scenario_3.yml (100%) rename examples/{fix_command => fix-command}/remove_scenarios/original_yaml_scenario_1.yml (100%) rename examples/{fix_command => fix-command}/remove_scenarios/original_yaml_scenario_2.yml (100%) rename examples/{fix_command => fix-command}/remove_scenarios/original_yaml_scenario_3.yml (100%) rename examples/{fix_command => fix-command}/replace_scenarios/fixed_yaml_scenario_1.yml (100%) rename examples/{fix_command => fix-command}/replace_scenarios/fixed_yaml_scenario_2.yml (100%) rename examples/{fix_command => fix-command}/replace_scenarios/original_yaml_scenario_1.yml (100%) rename examples/{fix_command => fix-command}/replace_scenarios/original_yaml_scenario_2.yml (100%) diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 9073361c..e3632015 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -1,6 +1,7 @@ package fixhandler import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -27,45 +28,124 @@ func NewFixHandlerMock() (*FixHandler, error) { }, nil } -func onlineBoutiquePath() string { +func fixCommandPath() string { o, _ := os.Getwd() - return filepath.Join(filepath.Dir(o), "..", "..", "examples", "online-boutique") + return filepath.Join(filepath.Dir(o), "..", "..", "examples", "fix-command") +} + +func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[]string, directoryPath string) { + + scenarioCount := len(*yamlExpressions) + + for scenario := 1; scenario <= scenarioCount; scenario++ { + originalFile := fmt.Sprintf("original_yaml_scenario_%d.yml", scenario) + fixedFile := fmt.Sprintf("fixed_yaml_scenario_%d.yml", scenario) + + originalFilePath := filepath.Join(directoryPath, originalFile) + fixedFilePath := filepath.Join(directoryPath, fixedFile) + + // create temp file + tempFile, err := ioutil.TempFile("", originalFile) + if err != nil { + panic(err) + } + defer os.Remove(tempFile.Name()) + + // read original file + originalFileContent, err := ioutil.ReadFile(originalFilePath) + if err != nil { + panic(err) + } + + // write original file contents to temp file + err = ioutil.WriteFile(tempFile.Name(), originalFileContent, 0644) + if err != nil { + panic(err) + } + + // make changes to temp file + h, _ := NewFixHandlerMock() + err = h.applyFixToFile(tempFile.Name(), (*yamlExpressions)[scenario]) + assert.NoError(t, err) + + // Check temp file contents + tempFileContent, err := ioutil.ReadFile(tempFile.Name()) + if err != nil { + panic(err) + } + + // Get fixed Yaml file contents + fixedFileContent, err := ioutil.ReadFile(fixedFilePath) + + assert.Equal(t, tempFileContent, fixedFileContent) + + } +} + +func testDirectoryApplyFix(t *testing.T, directory string) { + directoryPath := filepath.Join(fixCommandPath(), directory) + var yamlExpressions []string + + switch directory { + case "insert_scenarios": + yamlExpressions = []string{ + "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]", + + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", + + `select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation |= false | + select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"] | + select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type |= RuntimeDefault | + select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false | + select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true`, + + "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", + } + + case "remove_scenarios": + yamlExpressions = []string{ + "del(select(di==0).spec.containers[0].securityContext)", + + "del(select(di==0).spec.containers[1])", + + "del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1])", + } + + case "replace_scenarios": + yamlExpressions = []string{ + "select(di==0).spec.containers[0].securityContext.runAsRoot |= false", + + `select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" | + select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW"`, + } + + case "hybrid_scenarios": + yamlExpressions = []string{ + `del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`, + } + } + + testDirectoryApplyFixHelper(t, &yamlExpressions, directoryPath) } func TestFixHandler_applyFixToFile(t *testing.T) { - originalFilePath := filepath.Join(onlineBoutiquePath(), "adservice.yaml") - // create temp file - tempFile, err := ioutil.TempFile("", "adservice.yaml") - if err != nil { - panic(err) - } - defer os.Remove(tempFile.Name()) + // Tests for Insert scenarios + testDirectoryApplyFix(t, "insert_scenarios") - // read original file - b, err := ioutil.ReadFile(originalFilePath) - if err != nil { - panic(err) - } - assert.NotContains(t, string(b), "readOnlyRootFilesystem: true") + // Tests for Removal scenarios + testDirectoryApplyFix(t, "remove_scenarios") - // write original file contents to temp file - err = ioutil.WriteFile(tempFile.Name(), b, 0644) - if err != nil { - panic(err) - } + // Tests for Replace scenarios + testDirectoryApplyFix(t, "replace_scenarios") - // make changes to temp file - h, _ := NewFixHandlerMock() - yamlExpression := "select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true" - err = h.applyFixToFile(tempFile.Name(), yamlExpression) - assert.NoError(t, err) + // Tests for Hybrid Scenarios + testDirectoryApplyFix(t, "hybrid_scenarios") - // Check temp file contents - b, err = ioutil.ReadFile(tempFile.Name()) - if err != nil { - panic(err) - } - assert.Contains(t, string(b), "readOnlyRootFilesystem: true") } func Test_fixPathToValidYamlExpression(t *testing.T) { diff --git a/examples/fix_command/hybrid_scenarios/fixed_yaml_scenario_1.yml b/examples/fix-command/hybrid_scenarios/fixed_yaml_scenario_1.yml similarity index 100% rename from examples/fix_command/hybrid_scenarios/fixed_yaml_scenario_1.yml rename to examples/fix-command/hybrid_scenarios/fixed_yaml_scenario_1.yml diff --git a/examples/fix_command/hybrid_scenarios/original_yaml_scenario_1.yml b/examples/fix-command/hybrid_scenarios/original_yaml_scenario_1.yml similarity index 100% rename from examples/fix_command/hybrid_scenarios/original_yaml_scenario_1.yml rename to examples/fix-command/hybrid_scenarios/original_yaml_scenario_1.yml diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_1.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_1.yml similarity index 100% rename from examples/fix_command/insert_scenarios/fixed_yaml_scenario_1.yml rename to examples/fix-command/insert_scenarios/fixed_yaml_scenario_1.yml diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_2.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_2.yml similarity index 100% rename from examples/fix_command/insert_scenarios/fixed_yaml_scenario_2.yml rename to examples/fix-command/insert_scenarios/fixed_yaml_scenario_2.yml diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_3.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_3.yml similarity index 100% rename from examples/fix_command/insert_scenarios/fixed_yaml_scenario_3.yml rename to examples/fix-command/insert_scenarios/fixed_yaml_scenario_3.yml diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_4.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml similarity index 75% rename from examples/fix_command/insert_scenarios/fixed_yaml_scenario_4.yml rename to examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml index ebe4c8fc..f58931ae 100644 --- a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_4.yml +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml @@ -1,9 +1,9 @@ # Fixes to Apply: -# 1) spec.template.spec.securityContext.allowPrivilegeEscalation = false -# 2) spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] -# 3) spec.template.spec.containers[0].securityContext.seccompProfile.type = RuntimeDefault -# 4) spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation = false -# 5) spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem = true +# 1) select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation = false +# 2) select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] +# 3) select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type = RuntimeDefault +# 4) select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false +# 5) select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true apiVersion: apps/v1 kind: Deployment diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_5.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_5.yml similarity index 100% rename from examples/fix_command/insert_scenarios/fixed_yaml_scenario_5.yml rename to examples/fix-command/insert_scenarios/fixed_yaml_scenario_5.yml diff --git a/examples/fix_command/insert_scenarios/fixed_yaml_scenario_6.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml similarity index 100% rename from examples/fix_command/insert_scenarios/fixed_yaml_scenario_6.yml rename to examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_1.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_1.yml similarity index 100% rename from examples/fix_command/insert_scenarios/original_yaml_scenario_1.yml rename to examples/fix-command/insert_scenarios/original_yaml_scenario_1.yml diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_2.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_2.yml similarity index 100% rename from examples/fix_command/insert_scenarios/original_yaml_scenario_2.yml rename to examples/fix-command/insert_scenarios/original_yaml_scenario_2.yml diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_3.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_3.yml similarity index 100% rename from examples/fix_command/insert_scenarios/original_yaml_scenario_3.yml rename to examples/fix-command/insert_scenarios/original_yaml_scenario_3.yml diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_4.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_4.yml similarity index 70% rename from examples/fix_command/insert_scenarios/original_yaml_scenario_4.yml rename to examples/fix-command/insert_scenarios/original_yaml_scenario_4.yml index 4a4b896a..2c168f3f 100644 --- a/examples/fix_command/insert_scenarios/original_yaml_scenario_4.yml +++ b/examples/fix-command/insert_scenarios/original_yaml_scenario_4.yml @@ -1,9 +1,9 @@ # Fixes to Apply: -# 1) spec.template.spec.securityContext.allowPrivilegeEscalation = false -# 2) spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] -# 3) spec.template.spec.containers[0].securityContext.seccompProfile.type = RuntimeDefault -# 4) spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation = false -# 5) spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem = true +# 1) select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation = false +# 2) select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] +# 3) select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type = RuntimeDefault +# 4) select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false +# 5) select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true apiVersion: apps/v1 kind: Deployment diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_5.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_5.yml similarity index 100% rename from examples/fix_command/insert_scenarios/original_yaml_scenario_5.yml rename to examples/fix-command/insert_scenarios/original_yaml_scenario_5.yml diff --git a/examples/fix_command/insert_scenarios/original_yaml_scenario_6.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_6.yml similarity index 100% rename from examples/fix_command/insert_scenarios/original_yaml_scenario_6.yml rename to examples/fix-command/insert_scenarios/original_yaml_scenario_6.yml diff --git a/examples/fix_command/remove_scenarios/fixed_yaml_scenario_1.yml b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_1.yml similarity index 100% rename from examples/fix_command/remove_scenarios/fixed_yaml_scenario_1.yml rename to examples/fix-command/remove_scenarios/fixed_yaml_scenario_1.yml diff --git a/examples/fix_command/remove_scenarios/fixed_yaml_scenario_2.yml b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_2.yml similarity index 100% rename from examples/fix_command/remove_scenarios/fixed_yaml_scenario_2.yml rename to examples/fix-command/remove_scenarios/fixed_yaml_scenario_2.yml diff --git a/examples/fix_command/remove_scenarios/fixed_yaml_scenario_3.yml b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_3.yml similarity index 100% rename from examples/fix_command/remove_scenarios/fixed_yaml_scenario_3.yml rename to examples/fix-command/remove_scenarios/fixed_yaml_scenario_3.yml diff --git a/examples/fix_command/remove_scenarios/original_yaml_scenario_1.yml b/examples/fix-command/remove_scenarios/original_yaml_scenario_1.yml similarity index 100% rename from examples/fix_command/remove_scenarios/original_yaml_scenario_1.yml rename to examples/fix-command/remove_scenarios/original_yaml_scenario_1.yml diff --git a/examples/fix_command/remove_scenarios/original_yaml_scenario_2.yml b/examples/fix-command/remove_scenarios/original_yaml_scenario_2.yml similarity index 100% rename from examples/fix_command/remove_scenarios/original_yaml_scenario_2.yml rename to examples/fix-command/remove_scenarios/original_yaml_scenario_2.yml diff --git a/examples/fix_command/remove_scenarios/original_yaml_scenario_3.yml b/examples/fix-command/remove_scenarios/original_yaml_scenario_3.yml similarity index 100% rename from examples/fix_command/remove_scenarios/original_yaml_scenario_3.yml rename to examples/fix-command/remove_scenarios/original_yaml_scenario_3.yml diff --git a/examples/fix_command/replace_scenarios/fixed_yaml_scenario_1.yml b/examples/fix-command/replace_scenarios/fixed_yaml_scenario_1.yml similarity index 100% rename from examples/fix_command/replace_scenarios/fixed_yaml_scenario_1.yml rename to examples/fix-command/replace_scenarios/fixed_yaml_scenario_1.yml diff --git a/examples/fix_command/replace_scenarios/fixed_yaml_scenario_2.yml b/examples/fix-command/replace_scenarios/fixed_yaml_scenario_2.yml similarity index 100% rename from examples/fix_command/replace_scenarios/fixed_yaml_scenario_2.yml rename to examples/fix-command/replace_scenarios/fixed_yaml_scenario_2.yml diff --git a/examples/fix_command/replace_scenarios/original_yaml_scenario_1.yml b/examples/fix-command/replace_scenarios/original_yaml_scenario_1.yml similarity index 100% rename from examples/fix_command/replace_scenarios/original_yaml_scenario_1.yml rename to examples/fix-command/replace_scenarios/original_yaml_scenario_1.yml diff --git a/examples/fix_command/replace_scenarios/original_yaml_scenario_2.yml b/examples/fix-command/replace_scenarios/original_yaml_scenario_2.yml similarity index 100% rename from examples/fix_command/replace_scenarios/original_yaml_scenario_2.yml rename to examples/fix-command/replace_scenarios/original_yaml_scenario_2.yml From fa03a9dae39c03df595307bf4f196ab1a921a9fb Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Fri, 9 Dec 2022 18:55:31 +0530 Subject: [PATCH 11/39] Fixed: Comments and empty lines at the head are excluded --- core/pkg/fixhandler/fixhandler.go | 11 +++++- core/pkg/fixhandler/fixhandler_test.go | 4 +- core/pkg/fixhandler/yamlhandler.go | 5 ++- core/pkg/fixhandler/yamlhelper.go | 55 +++++++++++++++++++++++++- 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 78f39bae..d75b2bf8 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -216,13 +216,22 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str } func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError error) { + + // While obtaining fixedYamlNode, comments and empty lines at the top are ignored. In order to deal with that, + // comments and empty lines are removed and they are inserted again when applying fixes to file. + contentAtHead, err := truncateContentAtHead(filePath) + + if err != nil { + logger.L().Fatal("Error truncating comments and empty lines at head") + } originalYamlNode := getDecodedYaml(filePath) fixedYamlNode := getFixedYamlNode(filePath, yamlExpression) originalList := getDFSOrder(originalYamlNode) fixedList := getDFSOrder(fixedYamlNode) contentToAdd, linesToRemove := getFixInfo(originalList, fixedList) - err := applyFixesToFile(filePath, contentToAdd, linesToRemove) + + err = applyFixesToFile(filePath, contentToAdd, linesToRemove, contentAtHead) return err } diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index e3632015..59603a49 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -77,7 +77,9 @@ func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[]string, direct // Get fixed Yaml file contents fixedFileContent, err := ioutil.ReadFile(fixedFilePath) - assert.Equal(t, tempFileContent, fixedFileContent) + errorMessage := fmt.Sprintf("Content of fixed %s doesn't match content of %s in %s", originalFile, fixedFile, directoryPath) + + assert.Equal(t, tempFileContent, fixedFileContent, errorMessage) } } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index 5223e9a5..f51b16d2 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -184,7 +184,7 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { return updatedOriginalTracker, updatedFixedTracker } -func applyFixesToFile(filePath string, lineAndContentsToAdd *[]ContentToAdd, linesToRemove *[]ContentToRemove) (cmdError error) { +func applyFixesToFile(filePath string, lineAndContentsToAdd *[]ContentToAdd, linesToRemove *[]ContentToRemove, contentAtHead string) error { linesSlice, err := getLinesSlice(filePath) if err != nil { @@ -212,6 +212,9 @@ func applyFixesToFile(filePath string, lineAndContentsToAdd *[]ContentToAdd, lin writer := bufio.NewWriter(file) lineIdx, lineToAddIdx := 0, 0 + // Insert the comments and lines at the head removed initially. + writer.WriteString(contentAtHead) + for lineToAddIdx < len(*lineAndContentsToAdd) { for lineIdx <= (*lineAndContentsToAdd)[lineToAddIdx].Line { if linesSlice[lineIdx] == "*" { diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index f023a959..60b9135c 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -177,6 +177,7 @@ func enocodeIntoYaml(parentNode *yaml.Node, dfsOrder *[]NodeInfo, tracker int) ( } buf := new(bytes.Buffer) encoder := yaml.NewEncoder(buf) + encoder.SetIndent(2) errorEncoding := encoder.Encode(parentForContent) if errorEncoding != nil { return "", fmt.Errorf("Error debugging node, %v", errorEncoding.Error()) @@ -238,13 +239,65 @@ func isEmptyLineOrComment(lineContent string) bool { return false } +// Truncates the comments and empty lines at the top of the file and +// returns the truncated content +func truncateContentAtHead(filePath string) (string, error) { + var contentAtHead string + + linesSlice, err := getLinesSlice(filePath) + + if err != nil { + return "", err + } + + if err := os.Truncate(filePath, 0); err != nil { + return "", err + } + + file, err := os.OpenFile(filePath, os.O_RDWR, 0644) + if err != nil { + return "", err + } + + defer func() error { + if err := file.Close(); err != nil { + return err + } + return nil + }() + + lineIdx := 0 + + for lineIdx < len(linesSlice) { + if isEmptyLineOrComment(linesSlice[lineIdx]) { + contentAtHead += (linesSlice[lineIdx] + "\n") + lineIdx += 1 + } else { + break + } + } + + writer := bufio.NewWriter(file) + + for lineIdx < len(linesSlice) { + _, err = writer.WriteString(linesSlice[lineIdx] + "\n") + if err != nil { + return "", err + } + lineIdx += 1 + } + + writer.Flush() + return contentAtHead, nil +} + // Get the lines of existing yaml in a slice func getLinesSlice(filePath string) ([]string, error) { lineSlice := make([]string, 0) file, err := os.Open(filePath) if err != nil { - log.Fatal(err) + logger.L().Fatal(fmt.Sprintf("Cannot open file %s", filePath)) return nil, err } defer file.Close() From f72cb215d70786bc0068b0f5f64ca2be3adbc2a9 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Fri, 9 Dec 2022 22:45:55 +0530 Subject: [PATCH 12/39] Adjust Content Line --- core/pkg/fixhandler/fixhandler_test.go | 4 +-- core/pkg/fixhandler/yamlhandler.go | 38 +++++++++++++++++++------- core/pkg/fixhandler/yamlhelper.go | 1 + 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 59603a49..57155560 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -65,7 +65,7 @@ func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[]string, direct // make changes to temp file h, _ := NewFixHandlerMock() - err = h.applyFixToFile(tempFile.Name(), (*yamlExpressions)[scenario]) + err = h.applyFixToFile(tempFile.Name(), (*yamlExpressions)[scenario-1]) assert.NoError(t, err) // Check temp file contents @@ -79,7 +79,7 @@ func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[]string, direct errorMessage := fmt.Sprintf("Content of fixed %s doesn't match content of %s in %s", originalFile, fixedFile, directoryPath) - assert.Equal(t, tempFileContent, fixedFileContent, errorMessage) + assert.Equal(t, string(tempFileContent), string(fixedFileContent), errorMessage) } } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index f51b16d2..ab7efeb8 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -184,7 +184,21 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { return updatedOriginalTracker, updatedFixedTracker } -func applyFixesToFile(filePath string, lineAndContentsToAdd *[]ContentToAdd, linesToRemove *[]ContentToRemove, contentAtHead string) error { +// Line numbers are readjusted such that there are no empty lines or comment lines before them +func adjustContentLines(contentToAdd *[]ContentToAdd, linesSlice *[]string) { + for contentIdx, content := range *contentToAdd { + line := content.Line + for idx := line - 1; idx >= 0; idx-- { + if isEmptyLineOrComment((*linesSlice)[idx]) { + (*contentToAdd)[contentIdx].Line -= 1 + } else { + break + } + } + } +} + +func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemove *[]ContentToRemove, contentAtHead string) error { linesSlice, err := getLinesSlice(filePath) if err != nil { @@ -210,32 +224,36 @@ func applyFixesToFile(filePath string, lineAndContentsToAdd *[]ContentToAdd, lin removeLines(linesToRemove, &linesSlice) writer := bufio.NewWriter(file) - lineIdx, lineToAddIdx := 0, 0 + lineIdx, lineToAddIdx := 1, 0 // Insert the comments and lines at the head removed initially. writer.WriteString(contentAtHead) - for lineToAddIdx < len(*lineAndContentsToAdd) { - for lineIdx <= (*lineAndContentsToAdd)[lineToAddIdx].Line { - if linesSlice[lineIdx] == "*" { + // Ideally, new node is inserted at line before the next node in DFS order. But, when the previous line contains a + // comment or empty line, we need to insert new nodes before them. + adjustContentLines(contentToAdd, &linesSlice) + + for lineToAddIdx < len(*contentToAdd) { + for lineIdx <= (*contentToAdd)[lineToAddIdx].Line { + if linesSlice[lineIdx-1] == "*" { continue } - _, err := writer.WriteString(linesSlice[lineIdx] + "\n") + _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") if err != nil { return err } lineIdx += 1 } - writeContentToAdd(writer, (*lineAndContentsToAdd)[lineToAddIdx].Content) + writeContentToAdd(writer, (*contentToAdd)[lineToAddIdx].Content) lineToAddIdx += 1 } - for lineIdx < len(linesSlice) { - if linesSlice[lineIdx] == "*" { + for lineIdx <= len(linesSlice) { + if linesSlice[lineIdx-1] == "*" { continue } - _, err := writer.WriteString(linesSlice[lineIdx] + "\n") + _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") if err != nil { return err } diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index 60b9135c..8e1ecb96 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -231,6 +231,7 @@ func removeLines(linesToRemove *[]ContentToRemove, linesSlice *[]string) { // Checks if the line is empty or a comment func isEmptyLineOrComment(lineContent string) bool { + lineContent = strings.TrimSpace(lineContent) if lineContent == "" { return true } else if lineContent[0:1] == "#" { From f64200f42f4959ce9689726a9da692e9aa78bb2d Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Fri, 9 Dec 2022 23:44:04 +0530 Subject: [PATCH 13/39] Small Changes --- core/pkg/fixhandler/yamlhandler.go | 26 +++++++------------ core/pkg/fixhandler/yamlhelper.go | 21 +++++++++++++++ .../fixed_yaml_scenario_1.yml | 2 +- .../fixed_yaml_scenario_2.yml | 4 +-- .../fixed_yaml_scenario_3.yml | 4 +-- .../fixed_yaml_scenario_5.yml | 2 +- .../fixed_yaml_scenario_6.yml | 2 +- .../original_yaml_scenario_3.yml | 2 +- 8 files changed, 39 insertions(+), 24 deletions(-) diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index ab7efeb8..67b24688 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -6,6 +6,7 @@ import ( "container/list" "fmt" "io/ioutil" + "math" "os" logger "github.com/kubescape/go-logger" @@ -132,7 +133,7 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]Conten } for fixedListTracker < len(*fixedList) { - fixInfoMetadata.originalListTracker = len(*originalList) - 1 + fixInfoMetadata.originalListTracker = -1 fixInfoMetadata.fixedListTracker = fixedListTracker fixedListTracker = addLinesToInsert(fixInfoMetadata) } @@ -156,7 +157,14 @@ func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) int { // Adds the lines to insert and returns the updated fixedListTracker func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) int { currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] - lineToInsert := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker].node.Line - 1 + + var lineToInsert int + if fixInfoMetadata.originalListTracker == -1 { + lineToInsert = int(math.Inf(1)) + } else { + lineToInsert = (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker].node.Line - 1 + } + contentToInsert := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) newTracker := updateTracker(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) @@ -184,20 +192,6 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { return updatedOriginalTracker, updatedFixedTracker } -// Line numbers are readjusted such that there are no empty lines or comment lines before them -func adjustContentLines(contentToAdd *[]ContentToAdd, linesSlice *[]string) { - for contentIdx, content := range *contentToAdd { - line := content.Line - for idx := line - 1; idx >= 0; idx-- { - if isEmptyLineOrComment((*linesSlice)[idx]) { - (*contentToAdd)[contentIdx].Line -= 1 - } else { - break - } - } - } -} - func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemove *[]ContentToRemove, contentAtHead string) error { linesSlice, err := getLinesSlice(filePath) diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index 8e1ecb96..511fcb9d 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -292,6 +292,27 @@ func truncateContentAtHead(filePath string) (string, error) { return contentAtHead, nil } +func adjustContentLines(contentToAdd *[]ContentToAdd, linesSlice *[]string) { + for contentIdx, content := range *contentToAdd { + line := content.Line + + // Update Line number to last line if their value is math.Inf + if line == int(math.Inf(1)) { + (*contentToAdd)[contentIdx].Line = len(*linesSlice) + continue + } + + // Adjust line numbers such that there are no "empty lines or comment lines of next nodes" before them + for idx := line - 1; idx >= 0; idx-- { + if isEmptyLineOrComment((*linesSlice)[idx]) { + (*contentToAdd)[contentIdx].Line -= 1 + } else { + break + } + } + } +} + // Get the lines of existing yaml in a slice func getLinesSlice(filePath string) ([]string, error) { lineSlice := make([]string, 0) diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_1.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_1.yml index 55f3aeb5..dbe3c81e 100644 --- a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_1.yml +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_1.yml @@ -11,4 +11,4 @@ spec: - name: nginx_container image: nginx securityContext: - allowPrivilegeEscalation: false \ No newline at end of file + allowPrivilegeEscalation: false diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_2.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_2.yml index 6ffcf01a..f0d77a9f 100644 --- a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_2.yml +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_2.yml @@ -7,9 +7,9 @@ metadata: name: insert_list spec: containers: - - name: nginx1 + - name: nginx_container image: nginx securityContext: capabilities: drop: - - NET_RAW \ No newline at end of file + - NET_RAW diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_3.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_3.yml index 1dfcfd49..60b4f0ed 100644 --- a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_3.yml +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_3.yml @@ -7,10 +7,10 @@ metadata: name: insert_list spec: containers: - - name: nginx1 + - name: nginx_container image: nginx securityContext: capabilities: drop: - NET_RAW - - SYS_ADM \ No newline at end of file + - SYS_ADM diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_5.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_5.yml index c9917375..6b66f26b 100644 --- a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_5.yml +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_5.yml @@ -15,4 +15,4 @@ spec: # Testing if comments are retained as intended securityContext: - runAsRoot: false \ No newline at end of file + runAsRoot: false diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml index 6eea6f7f..36968eec 100644 --- a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml @@ -13,4 +13,4 @@ spec: capabilities: drop: - "NET_RAW" - - "SYS_ADM" \ No newline at end of file + - "SYS_ADM" diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_3.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_3.yml index ec8f58e9..3d6c873a 100644 --- a/examples/fix-command/insert_scenarios/original_yaml_scenario_3.yml +++ b/examples/fix-command/insert_scenarios/original_yaml_scenario_3.yml @@ -7,7 +7,7 @@ metadata: name: insert_list spec: containers: - - name: nginx1 + - name: nginx_container image: nginx securityContext: capabilities: From 99fa81e411067ee2f2c5239ff580b58e27cde3d6 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Sat, 10 Dec 2022 12:00:15 +0530 Subject: [PATCH 14/39] Fix Bugs Remove Scenario --- core/pkg/fixhandler/fixhandler.go | 1 + core/pkg/fixhandler/fixhandler_test.go | 10 +-- core/pkg/fixhandler/yamlhandler.go | 77 +++++++++++++------ core/pkg/fixhandler/yamlhelper.go | 51 +++++++++--- .../fixed_yaml_scenario_4.yml | 14 +--- .../original_yaml_scenario_4.yml | 13 ---- 6 files changed, 100 insertions(+), 66 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index d75b2bf8..63e17ce4 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -229,6 +229,7 @@ func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError e originalList := getDFSOrder(originalYamlNode) fixedList := getDFSOrder(fixedYamlNode) + contentToAdd, linesToRemove := getFixInfo(originalList, fixedList) err = applyFixesToFile(filePath, contentToAdd, linesToRemove, contentAtHead) diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 57155560..4ef1aa77 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -98,8 +98,8 @@ func testDirectoryApplyFix(t *testing.T, directory string) { "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", `select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation |= false | - select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"] | - select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type |= RuntimeDefault | + select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] | + select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type |= "RuntimeDefault" | select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false | select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true`, @@ -140,13 +140,13 @@ func TestFixHandler_applyFixToFile(t *testing.T) { testDirectoryApplyFix(t, "insert_scenarios") // Tests for Removal scenarios - testDirectoryApplyFix(t, "remove_scenarios") + // testDirectoryApplyFix(t, "remove_scenarios") // Tests for Replace scenarios - testDirectoryApplyFix(t, "replace_scenarios") + // testDirectoryApplyFix(t, "replace_scenarios") // Tests for Hybrid Scenarios - testDirectoryApplyFix(t, "hybrid_scenarios") + // testDirectoryApplyFix(t, "hybrid_scenarios") } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index 67b24688..149ac593 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -72,10 +72,8 @@ func matchNodes(nodeOne, nodeTwo *yaml.Node) int { isNewNode := nodeTwo.Line == 0 && nodeTwo.Column == 0 sameLines := nodeOne.Line == nodeTwo.Line sameColumns := nodeOne.Column == nodeTwo.Column - sameKinds := nodeOne.Kind == nodeTwo.Kind - sameValues := nodeOne.Value == nodeTwo.Value - isSameNode := sameKinds && sameValues && sameLines && sameColumns + isSameNode := isSameNode(nodeOne, nodeTwo) switch { case isSameNode: @@ -119,7 +117,7 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]Conten originalListTracker = addLinesToRemove(fixInfoMetadata) case int(insertedNode): - fixedListTracker = addLinesToInsert(fixInfoMetadata) + originalListTracker, fixedListTracker = addLinesToInsert(fixInfoMetadata) case int(replacedNode): originalListTracker, fixedListTracker = updateLinesToReplace(fixInfoMetadata) @@ -133,9 +131,9 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]Conten } for fixedListTracker < len(*fixedList) { - fixInfoMetadata.originalListTracker = -1 + fixInfoMetadata.originalListTracker = int(math.Inf(1)) fixInfoMetadata.fixedListTracker = fixedListTracker - fixedListTracker = addLinesToInsert(fixInfoMetadata) + _, fixedListTracker = addLinesToInsert(fixInfoMetadata) } return &contentToAdd, &linesToRemove @@ -155,11 +153,46 @@ func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) int { } // Adds the lines to insert and returns the updated fixedListTracker -func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) int { +func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] + isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) + + if isOneLine { + parentNode := (*fixInfoMetadata.fixedList)[parentTracker] + originalTracker := getTracker(fixInfoMetadata.originalList, &parentNode) + line := parentNode.node.Line + content := getContent(parentNode.parent, fixInfoMetadata.fixedList, parentTracker) + + // Remove the Single line + *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ + startLine: line, + endLine: line, + }) + + // Encode entire Sequence Node and Insert + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ + Line: line, + Content: content, + }) + + var newOriginalTracker int + + if fixInfoMetadata.originalListTracker != int(math.Inf(1)) { + newOriginalTracker = updateTracker(fixInfoMetadata.originalList, originalTracker) + + } else { + newOriginalTracker = int(math.Inf(1)) + } + + newFixedTracker := updateTracker(fixInfoMetadata.fixedList, parentTracker) + + return newOriginalTracker, newFixedTracker + + } + var lineToInsert int - if fixInfoMetadata.originalListTracker == -1 { + if fixInfoMetadata.originalListTracker == int(math.Inf(1)) { lineToInsert = int(math.Inf(1)) } else { lineToInsert = (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker].node.Line - 1 @@ -167,14 +200,14 @@ func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) int { contentToInsert := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) - newTracker := updateTracker(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) + newFixedTracker := updateTracker(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ Line: lineToInsert, Content: contentToInsert, }) - return newTracker + return fixInfoMetadata.originalListTracker, newFixedTracker } // Adds the lines to remove and insert and updates the fixedListTracker and originalListTracker @@ -187,7 +220,7 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { } updatedOriginalTracker := addLinesToRemove(fixInfoMetadata) - updatedFixedTracker := addLinesToInsert(fixInfoMetadata) + _, updatedFixedTracker := addLinesToInsert(fixInfoMetadata) return updatedOriginalTracker, updatedFixedTracker } @@ -229,12 +262,11 @@ func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemo for lineToAddIdx < len(*contentToAdd) { for lineIdx <= (*contentToAdd)[lineToAddIdx].Line { - if linesSlice[lineIdx-1] == "*" { - continue - } - _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") - if err != nil { - return err + if linesSlice[lineIdx-1] != "*" { + _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") + if err != nil { + return err + } } lineIdx += 1 } @@ -244,12 +276,11 @@ func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemo } for lineIdx <= len(linesSlice) { - if linesSlice[lineIdx-1] == "*" { - continue - } - _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") - if err != nil { - return err + if linesSlice[lineIdx-1] != "*" { + _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") + if err != nil { + return err + } } lineIdx += 1 } diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index 511fcb9d..a480ba2f 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -148,17 +148,26 @@ func getNodeLine(dfsOrder *[]NodeInfo, tracker int) int { } } -func isOneLineSequenceNode(node *yaml.Node) bool { - if node.Kind != yaml.SequenceNode { - return false +func isOneLineSequenceNode(list *[]NodeInfo, currentTracker int) (bool, int) { + parentNode := (*list)[currentTracker].parent + if parentNode.Kind != yaml.SequenceNode { + return false, -1 } - nodeLine := node.Line - for _, child := range node.Content { - if child.Line != nodeLine { - return false + + var currentNode, prevNode NodeInfo + currentTracker -= 1 + + for (*list)[currentTracker].node != parentNode { + currentNode = (*list)[currentTracker] + prevNode = (*list)[currentTracker-1] + + if currentNode.node.Line != prevNode.node.Line { + return false, -1 } + currentTracker -= 1 } - return true + + return true, currentTracker } func enocodeIntoYaml(parentNode *yaml.Node, dfsOrder *[]NodeInfo, tracker int) (string, error) { @@ -214,13 +223,22 @@ func indentContent(content string, indentationSpaces int) string { return indentedContent } +func getTracker(list *[]NodeInfo, node *NodeInfo) int { + tracker := 0 + + for !isSameNode((*list)[tracker].node, node.node) { + tracker += 1 + } + + return tracker +} + func removeLines(linesToRemove *[]ContentToRemove, linesSlice *[]string) { for _, lineToRemove := range *linesToRemove { - startLine := lineToRemove.startLine - endLine := int(math.Min(float64(lineToRemove.endLine), float64(len(*linesSlice)-1))) - + startLine := lineToRemove.startLine - 1 + endLine := int(math.Min(float64(lineToRemove.endLine), float64(len(*linesSlice)))) - 1 for line := startLine; line <= endLine; line++ { - lineContent := strings.ReplaceAll((*linesSlice)[line], " ", "") + lineContent := (*linesSlice)[line] if isEmptyLineOrComment(lineContent) { break } @@ -229,6 +247,15 @@ func removeLines(linesToRemove *[]ContentToRemove, linesSlice *[]string) { } } +func isSameNode(nodeOne, nodeTwo *yaml.Node) bool { + sameLines := nodeOne.Line == nodeTwo.Line + sameColumns := nodeOne.Column == nodeTwo.Column + sameKinds := nodeOne.Kind == nodeTwo.Kind + sameValues := nodeOne.Value == nodeTwo.Value + + return sameKinds && sameValues && sameLines && sameColumns +} + // Checks if the line is empty or a comment func isEmptyLineOrComment(lineContent string) bool { lineContent = strings.TrimSpace(lineContent) diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml index f58931ae..8081737f 100644 --- a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml @@ -55,16 +55,4 @@ spec: readOnlyRootFilesystem: true securityContext: allowPrivilegeEscalation: false ---- -apiVersion: v1 -kind: Service -metadata: - name: example_4 -spec: - type: ClusterIP - selector: - app: example_4 - ports: - - name: test_port - port: 3000 - targetPort: 3000 + diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_4.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_4.yml index 2c168f3f..5ef36f1e 100644 --- a/examples/fix-command/insert_scenarios/original_yaml_scenario_4.yml +++ b/examples/fix-command/insert_scenarios/original_yaml_scenario_4.yml @@ -45,16 +45,3 @@ spec: periodSeconds: 15 exec: command: ["/bin/grpc_health_probe", "-addr=:3000"] ---- -apiVersion: v1 -kind: Service -metadata: - name: example_4 -spec: - type: ClusterIP - selector: - app: example_4 - ports: - - name: test_port - port: 3000 - targetPort: 3000 From baf62887b9ab68b0fce16c5e7a6e543cc56c6284 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Sat, 10 Dec 2022 17:29:04 +0530 Subject: [PATCH 15/39] Handle Single Line Sequence Node --- core/pkg/fixhandler/yamlhandler.go | 25 ++++++--------- core/pkg/fixhandler/yamlhelper.go | 31 ++++++++++++++++++- .../fixed_yaml_scenario_4.yml | 3 +- .../fixed_yaml_scenario_6.yml | 4 +-- .../original_yaml_scenario_6.yml | 2 +- 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index 149ac593..a1f7a318 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -159,10 +159,11 @@ func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) if isOneLine { - parentNode := (*fixInfoMetadata.fixedList)[parentTracker] - originalTracker := getTracker(fixInfoMetadata.originalList, &parentNode) - line := parentNode.node.Line - content := getContent(parentNode.parent, fixInfoMetadata.fixedList, parentTracker) + originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) + fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) + + currentDFSNode = (*fixInfoMetadata.fixedList)[fixedListTracker] + content := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) // Remove the Single line *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ @@ -176,18 +177,10 @@ func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { Content: content, }) - var newOriginalTracker int + originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) + fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) - if fixInfoMetadata.originalListTracker != int(math.Inf(1)) { - newOriginalTracker = updateTracker(fixInfoMetadata.originalList, originalTracker) - - } else { - newOriginalTracker = int(math.Inf(1)) - } - - newFixedTracker := updateTracker(fixInfoMetadata.fixedList, parentTracker) - - return newOriginalTracker, newFixedTracker + return originalListTracker, fixedListTracker } @@ -220,7 +213,7 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { } updatedOriginalTracker := addLinesToRemove(fixInfoMetadata) - _, updatedFixedTracker := addLinesToInsert(fixInfoMetadata) + updatedOriginalTracker, updatedFixedTracker := addLinesToInsert(fixInfoMetadata) return updatedOriginalTracker, updatedFixedTracker } diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index a480ba2f..64f55f86 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -148,6 +148,7 @@ func getNodeLine(dfsOrder *[]NodeInfo, tracker int) int { } } +// Checks if the node is part of single line sequence node and returns the line func isOneLineSequenceNode(list *[]NodeInfo, currentTracker int) (bool, int) { parentNode := (*list)[currentTracker].parent if parentNode.Kind != yaml.SequenceNode { @@ -167,7 +168,22 @@ func isOneLineSequenceNode(list *[]NodeInfo, currentTracker int) (bool, int) { currentTracker -= 1 } - return true, currentTracker + parentNodeInfo := (*list)[currentTracker] + + if parentNodeInfo.parent.Kind == yaml.MappingNode { + keyNodeInfo := (*list)[currentTracker-1] + if keyNodeInfo.node.Line == parentNode.Line { + return true, parentNode.Line + } else { + return false, -1 + } + } else { + if parentNodeInfo.parent.Line == parentNode.Line { + return true, parentNode.Line + } else { + return false, -1 + } + } } func enocodeIntoYaml(parentNode *yaml.Node, dfsOrder *[]NodeInfo, tracker int) (string, error) { @@ -233,6 +249,19 @@ func getTracker(list *[]NodeInfo, node *NodeInfo) int { return tracker } +// Returns the first node in the given line that is not mapping node +func getFirstNodeInLine(list *[]NodeInfo, line int) int { + tracker := 0 + + currentNode := (*list)[tracker].node + for currentNode.Line != line || currentNode.Kind == yaml.MappingNode { + tracker += 1 + currentNode = (*list)[tracker].node + } + + return tracker +} + func removeLines(linesToRemove *[]ContentToRemove, linesSlice *[]string) { for _, lineToRemove := range *linesToRemove { startLine := lineToRemove.startLine - 1 diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml index 8081737f..9827672e 100644 --- a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml @@ -48,11 +48,10 @@ spec: securityContext: capabilities: drop: - - "NET_RAW" + - NET_RAW seccompProfile: type: RuntimeDefault allowPrivilegeEscalation: false readOnlyRootFilesystem: true securityContext: allowPrivilegeEscalation: false - diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml index 36968eec..b0091e62 100644 --- a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml @@ -11,6 +11,4 @@ spec: image: nginx securityContext: capabilities: - drop: - - "NET_RAW" - - "SYS_ADM" + drop: [NET_RAW, SYS_ADM] diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_6.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_6.yml index 383e5a45..a1638d45 100644 --- a/examples/fix-command/insert_scenarios/original_yaml_scenario_6.yml +++ b/examples/fix-command/insert_scenarios/original_yaml_scenario_6.yml @@ -11,4 +11,4 @@ spec: image: nginx securityContext: capabilities: - drop: ["NET_RAW"] \ No newline at end of file + drop: [NET_RAW] \ No newline at end of file From e02086e90cf4024dbb34f70a9fc61df633e0475f Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Sat, 10 Dec 2022 18:11:17 +0530 Subject: [PATCH 16/39] Handle Single Line Sequence Node in Removal --- core/pkg/fixhandler/fixhandler_test.go | 2 +- core/pkg/fixhandler/yamlhandler.go | 41 +++++++++++++++---- .../fixed_yaml_scenario_1.yml | 2 +- .../fixed_yaml_scenario_2.yml | 3 +- .../fixed_yaml_scenario_3.yml | 3 +- .../original_yaml_scenario_2.yml | 2 +- 6 files changed, 40 insertions(+), 13 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 4ef1aa77..cb54a415 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -140,7 +140,7 @@ func TestFixHandler_applyFixToFile(t *testing.T) { testDirectoryApplyFix(t, "insert_scenarios") // Tests for Removal scenarios - // testDirectoryApplyFix(t, "remove_scenarios") + testDirectoryApplyFix(t, "remove_scenarios") // Tests for Replace scenarios // testDirectoryApplyFix(t, "replace_scenarios") diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index a1f7a318..3ed45e27 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -114,7 +114,7 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]Conten fixedListTracker += 1 case int(removedNode): - originalListTracker = addLinesToRemove(fixInfoMetadata) + originalListTracker, fixedListTracker = addLinesToRemove(fixInfoMetadata) case int(insertedNode): originalListTracker, fixedListTracker = addLinesToInsert(fixInfoMetadata) @@ -127,7 +127,7 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]Conten for originalListTracker < len(*originalList) { fixInfoMetadata.originalListTracker = originalListTracker fixInfoMetadata.fixedListTracker = len(*fixedList) - 1 - originalListTracker = addLinesToRemove(fixInfoMetadata) + originalListTracker, _ = addLinesToRemove(fixInfoMetadata) } for fixedListTracker < len(*fixedList) { @@ -141,15 +141,43 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]Conten } // Adds the lines to remove and returns the updated originalListTracker -func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) int { +func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) (int, int) { currentDFSNode := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker] + + isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) + + if isOneLine { + originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) + fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) + + currentDFSNode = (*fixInfoMetadata.fixedList)[fixedListTracker] + content := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) + + // Remove the Single line + *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ + startLine: line, + endLine: line, + }) + + // Encode entire Sequence Node and Insert + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ + Line: line, + Content: content, + }) + + originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) + fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) + + return originalListTracker, fixedListTracker + } + newTracker := updateTracker(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ startLine: currentDFSNode.node.Line, endLine: getNodeLine(fixInfoMetadata.originalList, newTracker) - 1, }) - return newTracker + return newTracker, fixInfoMetadata.fixedListTracker } // Adds the lines to insert and returns the updated fixedListTracker @@ -181,7 +209,6 @@ func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) return originalListTracker, fixedListTracker - } var lineToInsert int @@ -212,8 +239,8 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { fixInfoMetadata.fixedListTracker -= 1 } - updatedOriginalTracker := addLinesToRemove(fixInfoMetadata) - updatedOriginalTracker, updatedFixedTracker := addLinesToInsert(fixInfoMetadata) + updatedOriginalTracker, updatedFixedTracker := addLinesToRemove(fixInfoMetadata) + updatedOriginalTracker, updatedFixedTracker = addLinesToInsert(fixInfoMetadata) return updatedOriginalTracker, updatedFixedTracker } diff --git a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_1.yml b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_1.yml index 8ecc8724..5c5f896c 100644 --- a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_1.yml +++ b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_1.yml @@ -9,4 +9,4 @@ metadata: spec: containers: - name: nginx_container - image: nginx \ No newline at end of file + image: nginx diff --git a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_2.yml b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_2.yml index 55cf481a..43b80fa1 100644 --- a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_2.yml +++ b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_2.yml @@ -9,4 +9,5 @@ metadata: spec: containers: - name: nginx_container - image: nginx \ No newline at end of file + image: nginx + diff --git a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_3.yml b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_3.yml index 254affe4..bc2225bd 100644 --- a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_3.yml +++ b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_3.yml @@ -11,5 +11,4 @@ spec: image: nginx securityContext: capabilities: - drop: - - NET_RAW \ No newline at end of file + drop: ["NET_RAW"] diff --git a/examples/fix-command/remove_scenarios/original_yaml_scenario_2.yml b/examples/fix-command/remove_scenarios/original_yaml_scenario_2.yml index 4b1f39d5..29ebfbf9 100644 --- a/examples/fix-command/remove_scenarios/original_yaml_scenario_2.yml +++ b/examples/fix-command/remove_scenarios/original_yaml_scenario_2.yml @@ -10,6 +10,6 @@ spec: containers: - name: nginx_container image: nginx - + - name: container_with_security_issues image: image_with_security_issues \ No newline at end of file From 17c52bd0aebfb05779c552f58442cea32bb12231 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Sat, 10 Dec 2022 18:24:06 +0530 Subject: [PATCH 17/39] Handle Single Line Sequence Node in Replacement --- core/pkg/fixhandler/fixhandler_test.go | 4 +-- core/pkg/fixhandler/yamlhandler.go | 27 +++++++++++++++++++ .../fixed_yaml_scenario_1.yml | 2 +- .../fixed_yaml_scenario_1.yml | 2 +- .../fixed_yaml_scenario_2.yml | 5 ++-- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index cb54a415..d5a2e76c 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -143,10 +143,10 @@ func TestFixHandler_applyFixToFile(t *testing.T) { testDirectoryApplyFix(t, "remove_scenarios") // Tests for Replace scenarios - // testDirectoryApplyFix(t, "replace_scenarios") + testDirectoryApplyFix(t, "replace_scenarios") // Tests for Hybrid Scenarios - // testDirectoryApplyFix(t, "hybrid_scenarios") + testDirectoryApplyFix(t, "hybrid_scenarios") } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index 3ed45e27..d4c608fe 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -234,6 +234,33 @@ func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] + isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) + + if isOneLine { + originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) + fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) + + currentDFSNode = (*fixInfoMetadata.fixedList)[fixedListTracker] + content := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) + + // Remove the Single line + *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ + startLine: line, + endLine: line, + }) + + // Encode entire Sequence Node and Insert + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ + Line: line, + Content: content, + }) + + originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) + fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) + + return originalListTracker, fixedListTracker + } + if isValueNodeinMapping(¤tDFSNode) { fixInfoMetadata.originalListTracker -= 1 fixInfoMetadata.fixedListTracker -= 1 diff --git a/examples/fix-command/hybrid_scenarios/fixed_yaml_scenario_1.yml b/examples/fix-command/hybrid_scenarios/fixed_yaml_scenario_1.yml index 5ab9f08c..b74c6cdc 100644 --- a/examples/fix-command/hybrid_scenarios/fixed_yaml_scenario_1.yml +++ b/examples/fix-command/hybrid_scenarios/fixed_yaml_scenario_1.yml @@ -16,4 +16,4 @@ spec: - name: nginx_container image: nginx securityContext: - runAsRoot: false \ No newline at end of file + runAsRoot: false diff --git a/examples/fix-command/replace_scenarios/fixed_yaml_scenario_1.yml b/examples/fix-command/replace_scenarios/fixed_yaml_scenario_1.yml index 4a4eda33..067565ca 100644 --- a/examples/fix-command/replace_scenarios/fixed_yaml_scenario_1.yml +++ b/examples/fix-command/replace_scenarios/fixed_yaml_scenario_1.yml @@ -11,4 +11,4 @@ spec: - name: nginx_container image: nginx securityContext: - runAsRoot: false \ No newline at end of file + runAsRoot: false diff --git a/examples/fix-command/replace_scenarios/fixed_yaml_scenario_2.yml b/examples/fix-command/replace_scenarios/fixed_yaml_scenario_2.yml index 89775e04..2c8b1da4 100644 --- a/examples/fix-command/replace_scenarios/fixed_yaml_scenario_2.yml +++ b/examples/fix-command/replace_scenarios/fixed_yaml_scenario_2.yml @@ -14,6 +14,5 @@ spec: securityContext: capabilities: drop: - - "SYS_ADM" - add: - - "NET_RAW" \ No newline at end of file + - "SYS_ADM" + add: ["NET_RAW"] From 36b7b8e2ac3a693c9fe6b5ecc5c955711d34daf3 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Sun, 11 Dec 2022 15:30:51 +0530 Subject: [PATCH 18/39] Replace Single Line Sequence Node --- core/pkg/fixhandler/yamlhandler.go | 71 +++--------------------------- core/pkg/fixhandler/yamlhelper.go | 27 ++++++++++++ 2 files changed, 32 insertions(+), 66 deletions(-) diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index d4c608fe..e828ffbd 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -147,28 +147,9 @@ func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) (int, int) { isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) if isOneLine { - originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) - fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) - - currentDFSNode = (*fixInfoMetadata.fixedList)[fixedListTracker] - content := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) - - // Remove the Single line - *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ - startLine: line, - endLine: line, - }) - - // Encode entire Sequence Node and Insert - *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ - Line: line, - Content: content, - }) - - originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) - fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) - - return originalListTracker, fixedListTracker + // Remove the entire line and replace it with the sequence node in fixed info. This way, + // the original formatting is lost. + return replaceSingleLineSequence(fixInfoMetadata, line) } newTracker := updateTracker(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) @@ -187,28 +168,7 @@ func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) if isOneLine { - originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) - fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) - - currentDFSNode = (*fixInfoMetadata.fixedList)[fixedListTracker] - content := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) - - // Remove the Single line - *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ - startLine: line, - endLine: line, - }) - - // Encode entire Sequence Node and Insert - *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ - Line: line, - Content: content, - }) - - originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) - fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) - - return originalListTracker, fixedListTracker + return replaceSingleLineSequence(fixInfoMetadata, line) } var lineToInsert int @@ -237,28 +197,7 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) if isOneLine { - originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) - fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) - - currentDFSNode = (*fixInfoMetadata.fixedList)[fixedListTracker] - content := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) - - // Remove the Single line - *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ - startLine: line, - endLine: line, - }) - - // Encode entire Sequence Node and Insert - *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ - Line: line, - Content: content, - }) - - originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) - fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) - - return originalListTracker, fixedListTracker + return replaceSingleLineSequence(fixInfoMetadata, line) } if isValueNodeinMapping(¤tDFSNode) { diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index 64f55f86..0ed4a596 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -393,6 +393,33 @@ func getLinesSlice(filePath string) ([]string, error) { return lineSlice, err } +// Remove the entire line and replace it with the sequence node in fixed info. This way, +// the original formatting is lost. +func replaceSingleLineSequence(fixInfoMetadata *FixInfoMetadata, line int) (int, int) { + originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) + fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) + + currentDFSNode := (*fixInfoMetadata.fixedList)[fixedListTracker] + content := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) + + // Remove the Single line + *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ + startLine: line, + endLine: line, + }) + + // Encode entire Sequence Node and Insert + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ + Line: line, + Content: content, + }) + + originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) + fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) + + return originalListTracker, fixedListTracker +} + func writeContentToAdd(writer *bufio.Writer, contentToAdd string) { scanner := bufio.NewScanner(strings.NewReader(contentToAdd)) for scanner.Scan() { From af5cdefc5f21d5929d88873d6f21af827fdd3754 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Sun, 11 Dec 2022 16:35:40 +0530 Subject: [PATCH 19/39] Improve Readability of code --- core/pkg/fixhandler/datastructures.go | 6 +- core/pkg/fixhandler/fixhandler.go | 8 +- core/pkg/fixhandler/fixhandler_test.go | 2 +- core/pkg/fixhandler/yamlhandler.go | 57 +-- core/pkg/fixhandler/yamlhelper.go | 515 ++++++++++++------------- 5 files changed, 294 insertions(+), 294 deletions(-) diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index 50f69d2e..cb40c060 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -39,7 +39,7 @@ type FixInfoMetadata struct { originalListTracker int fixedListTracker int contentToAdd *[]ContentToAdd - contentToRemove *[]ContentToRemove + linesToRemove *[]LinesToRemove } // ContentToAdd holds the information about where to insert the new changes in the existing yaml file @@ -50,8 +50,8 @@ type ContentToAdd struct { Content string } -// ContentToRemove holds the line numbers to remove from the existing yaml file -type ContentToRemove struct { +// LinesToRemove holds the line numbers to remove from the existing yaml file +type LinesToRemove struct { startLine int endLine int } diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 63e17ce4..ec00a975 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -224,11 +224,11 @@ func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError e if err != nil { logger.L().Fatal("Error truncating comments and empty lines at head") } - originalYamlNode := getDecodedYaml(filePath) - fixedYamlNode := getFixedYamlNode(filePath, yamlExpression) + originalYamlNode := constructDecodedYaml(filePath) + fixedYamlNode := constructFixedYamlNode(filePath, yamlExpression) - originalList := getDFSOrder(originalYamlNode) - fixedList := getDFSOrder(fixedYamlNode) + originalList := constructDFSOrder(originalYamlNode) + fixedList := constructDFSOrder(fixedYamlNode) contentToAdd, linesToRemove := getFixInfo(originalList, fixedList) diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index d5a2e76c..b8b08028 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -74,7 +74,7 @@ func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[]string, direct panic(err) } - // Get fixed Yaml file contents + // Get fixed Yaml file content and check if it is equal to tempFileContent fixedFileContent, err := ioutil.ReadFile(fixedFilePath) errorMessage := fmt.Sprintf("Content of fixed %s doesn't match content of %s in %s", originalFile, fixedFile, directoryPath) diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index e828ffbd..d52f4ea6 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -15,7 +15,7 @@ import ( "gopkg.in/yaml.v3" ) -func getDecodedYaml(filepath string) *yaml.Node { +func constructDecodedYaml(filepath string) *yaml.Node { file, err := ioutil.ReadFile(filepath) if err != nil { logger.L().Fatal("Cannot read file") @@ -33,13 +33,13 @@ func getDecodedYaml(filepath string) *yaml.Node { return &node } -func getFixedYamlNode(filePath, yamlExpression string) *yaml.Node { +func constructFixedYamlNode(filePath, yamlExpression string) *yaml.Node { preferences := yqlib.ConfiguredYamlPreferences preferences.EvaluateTogether = true decoder := yqlib.NewYamlDecoder(preferences) var allDocuments = list.New() - reader, err := getNewReader(filePath) + reader, err := constructNewReader(filePath) if err != nil { return &yaml.Node{} } @@ -61,9 +61,9 @@ func getFixedYamlNode(filePath, yamlExpression string) *yaml.Node { return matches.Front().Value.(*yqlib.CandidateNode).Node } -func getDFSOrder(node *yaml.Node) *[]NodeInfo { +func constructDFSOrder(node *yaml.Node) *[]NodeInfo { dfsOrder := make([]NodeInfo, 0) - getDFSOrderHelper(node, nil, &dfsOrder, 0) + constructDFSOrderHelper(node, nil, &dfsOrder, 0) return &dfsOrder } @@ -87,9 +87,9 @@ func matchNodes(nodeOne, nodeTwo *yaml.Node) int { } } -func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]ContentToRemove) { +func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]LinesToRemove) { contentToAdd := make([]ContentToAdd, 0) - linesToRemove := make([]ContentToRemove, 0) + linesToRemove := make([]LinesToRemove, 0) originalListTracker, fixedListTracker := 0, 0 @@ -99,7 +99,7 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]Conten originalListTracker: originalListTracker, fixedListTracker: fixedListTracker, contentToAdd: &contentToAdd, - contentToRemove: &linesToRemove, + linesToRemove: &linesToRemove, } for originalListTracker < len(*originalList) && fixedListTracker < len(*fixedList) { @@ -124,12 +124,13 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]Conten } } + // Some nodes are still not visited if they are removed at the end of the list for originalListTracker < len(*originalList) { fixInfoMetadata.originalListTracker = originalListTracker - fixInfoMetadata.fixedListTracker = len(*fixedList) - 1 originalListTracker, _ = addLinesToRemove(fixInfoMetadata) } + // Some nodes are still not visited if they are inserted at the end of the list for fixedListTracker < len(*fixedList) { fixInfoMetadata.originalListTracker = int(math.Inf(1)) fixInfoMetadata.fixedListTracker = fixedListTracker @@ -142,8 +143,6 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]Conten // Adds the lines to remove and returns the updated originalListTracker func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) (int, int) { - currentDFSNode := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker] - isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) if isOneLine { @@ -152,18 +151,19 @@ func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) (int, int) { return replaceSingleLineSequence(fixInfoMetadata, line) } - newTracker := updateTracker(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) - *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ + currentDFSNode := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker] + + newOriginalListTracker := updateTracker(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) + *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, LinesToRemove{ startLine: currentDFSNode.node.Line, - endLine: getNodeLine(fixInfoMetadata.originalList, newTracker) - 1, + endLine: getNodeLine(fixInfoMetadata.originalList, newOriginalListTracker) - 1, }) - return newTracker, fixInfoMetadata.fixedListTracker + return newOriginalListTracker, fixInfoMetadata.fixedListTracker } // Adds the lines to insert and returns the updated fixedListTracker func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { - currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) @@ -171,14 +171,10 @@ func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { return replaceSingleLineSequence(fixInfoMetadata, line) } - var lineToInsert int - if fixInfoMetadata.originalListTracker == int(math.Inf(1)) { - lineToInsert = int(math.Inf(1)) - } else { - lineToInsert = (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker].node.Line - 1 - } + currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] - contentToInsert := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) + lineToInsert := getLineToInsert(fixInfoMetadata) + contentToInsert := constructContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) newFixedTracker := updateTracker(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) @@ -192,7 +188,6 @@ func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { // Adds the lines to remove and insert and updates the fixedListTracker and originalListTracker func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { - currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) @@ -200,18 +195,21 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { return replaceSingleLineSequence(fixInfoMetadata, line) } + currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] + + // If only the value node is changed, entire "key-value" pair is replaced if isValueNodeinMapping(¤tDFSNode) { fixInfoMetadata.originalListTracker -= 1 fixInfoMetadata.fixedListTracker -= 1 } - updatedOriginalTracker, updatedFixedTracker := addLinesToRemove(fixInfoMetadata) - updatedOriginalTracker, updatedFixedTracker = addLinesToInsert(fixInfoMetadata) + addLinesToRemove(fixInfoMetadata) + updatedOriginalTracker, updatedFixedTracker := addLinesToInsert(fixInfoMetadata) return updatedOriginalTracker, updatedFixedTracker } -func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemove *[]ContentToRemove, contentAtHead string) error { +func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemove *[]LinesToRemove, contentAtHead string) error { linesSlice, err := getLinesSlice(filePath) if err != nil { @@ -248,6 +246,7 @@ func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemo for lineToAddIdx < len(*contentToAdd) { for lineIdx <= (*contentToAdd)[lineToAddIdx].Line { + // Check if the current line is not removed if linesSlice[lineIdx-1] != "*" { _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") if err != nil { @@ -257,7 +256,9 @@ func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemo lineIdx += 1 } - writeContentToAdd(writer, (*contentToAdd)[lineToAddIdx].Content) + content := (*contentToAdd)[lineToAddIdx].Content + writer.WriteString(content) + lineToAddIdx += 1 } diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index 0ed4a596..502c48a1 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -24,7 +24,41 @@ const ( replacedNode ) -func getNewReader(filename string) (io.Reader, error) { +func adjustContentLines(contentToAdd *[]ContentToAdd, linesSlice *[]string) { + for contentIdx, content := range *contentToAdd { + line := content.Line + + // Update Line number to last line if their value is math.Inf + if line == int(math.Inf(1)) { + (*contentToAdd)[contentIdx].Line = len(*linesSlice) + continue + } + + // Adjust line numbers such that there are no "empty lines or comment lines of next nodes" before them + for idx := line - 1; idx >= 0; idx-- { + if isEmptyLineOrComment((*linesSlice)[idx]) { + (*contentToAdd)[contentIdx].Line -= 1 + } else { + break + } + } + } +} + +func constructDFSOrderHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]NodeInfo, index int) { + dfsNode := NodeInfo{ + node: node, + parent: parent, + index: index, + } + *dfsOrder = append(*dfsOrder, dfsNode) + + for idx, child := range node.Content { + constructDFSOrderHelper(child, node, dfsOrder, idx) + } +} + +func constructNewReader(filename string) (io.Reader, error) { var reader *bufio.Reader if filename == "-" { reader = bufio.NewReader(os.Stdin) @@ -40,114 +74,114 @@ func getNewReader(filename string) (io.Reader, error) { return reader, nil } -func readDocuments(reader io.Reader, filename string, fileIndex int, decoder yqlib.Decoder) (*list.List, error) { - err := decoder.Init(reader) +func enocodeIntoYaml(parentNode *yaml.Node, nodeList *[]NodeInfo, tracker int) (string, error) { + content := make([]*yaml.Node, 0) + currentNode := (*nodeList)[tracker].node + content = append(content, currentNode) + + // Add the value in "key-value" pair to construct if the parent is mapping node + if parentNode.Kind == yaml.MappingNode { + valueNode := (*nodeList)[tracker+1].node + content = append(content, valueNode) + } + + // The parent is added at the top to encode into YAML + parentForContent := yaml.Node{ + Kind: parentNode.Kind, + Content: content, + } + + buf := new(bytes.Buffer) + + encoder := yaml.NewEncoder(buf) + encoder.SetIndent(2) + + errorEncoding := encoder.Encode(parentForContent) + if errorEncoding != nil { + return "", fmt.Errorf("Error debugging node, %v", errorEncoding.Error()) + } + errorClosingEncoder := encoder.Close() + if errorClosingEncoder != nil { + return "", fmt.Errorf("Error closing encoder: %v", errorClosingEncoder.Error()) + } + return fmt.Sprintf(`%v`, buf.String()), nil +} + +func constructContent(parentNode *yaml.Node, nodeList *[]NodeInfo, tracker int) string { + content, err := enocodeIntoYaml(parentNode, nodeList, tracker) if err != nil { + logger.L().Fatal("Cannot Encode into YAML") + } + + indentationSpaces := parentNode.Column - 1 + + content = indentContent(content, indentationSpaces) + + return content +} + +func indentContent(content string, indentationSpaces int) string { + indentedContent := "" + indentSpaces := strings.Repeat(" ", indentationSpaces) + + scanner := bufio.NewScanner(strings.NewReader(content)) + for scanner.Scan() { + line := scanner.Text() + indentedContent += (indentSpaces + line + "\n") + } + return indentedContent +} + +// Get the lines of existing yaml in a slice +func getLinesSlice(filePath string) ([]string, error) { + lineSlice := make([]string, 0) + + file, err := os.Open(filePath) + if err != nil { + logger.L().Fatal(fmt.Sprintf("Cannot open file %s", filePath)) return nil, err } - inputList := list.New() - var currentIndex uint + defer file.Close() - for { - candidateNode, errorReading := decoder.Decode() + scanner := bufio.NewScanner(file) - if errors.Is(errorReading, io.EOF) { - switch reader := reader.(type) { - case *os.File: - safelyCloseFile(reader) - } - return inputList, nil - } else if errorReading != nil { - return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading) - } - candidateNode.Document = currentIndex - candidateNode.Filename = filename - candidateNode.FileIndex = fileIndex - candidateNode.EvaluateTogether = true - - inputList.PushBack(candidateNode) - - currentIndex = currentIndex + 1 + for scanner.Scan() { + lineSlice = append(lineSlice, scanner.Text()) } + if err := scanner.Err(); err != nil { + log.Fatal(err) + return nil, err + } + + return lineSlice, err } -func safelyCloseFile(file *os.File) { - err := file.Close() - if err != nil { - logger.L().Error("Error Closing File") - } -} - -func getDFSOrderHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]NodeInfo, index int) { - dfsNode := NodeInfo{ - node: node, - parent: parent, - index: index, - } - *dfsOrder = append(*dfsOrder, dfsNode) - - for idx, child := range node.Content { - getDFSOrderHelper(child, node, dfsOrder, idx) - } -} - -// Skips the current node including it's children in DFS order and returns the new tracker. -func skipCurrentNode(node *yaml.Node, currentTracker int) int { - updatedTracker := currentTracker + getChildrenCount(node) - return updatedTracker -} - -func getChildrenCount(node *yaml.Node) int { - totalChildren := 1 - for _, child := range node.Content { - totalChildren += getChildrenCount(child) - } - return totalChildren -} - -// Moves the tracker to the parent of given node -func traceBackToParent(dfsOrder *[]NodeInfo, currentTracker int) int { - parentNode := (*dfsOrder)[currentTracker].parent - parentIdx := currentTracker - 1 - for parentIdx >= 0 { - if (*dfsOrder)[parentIdx].node == parentNode { - return parentIdx - } - parentIdx -= 1 - } - return 0 -} - -// Checks if the node is value node in "key-value" pairs of mapping node -func isValueNodeinMapping(dfsNode *NodeInfo) bool { - if dfsNode.parent.Kind == yaml.MappingNode && dfsNode.index%2 != 0 { - return true - } - return false -} - -func updateTracker(dfsOrder *[]NodeInfo, tracker int) int { - currentDFSNode := (*dfsOrder)[tracker] - var newTracker int - - if currentDFSNode.parent.Kind == yaml.MappingNode { - valueNode := (*dfsOrder)[tracker+1] - newTracker = skipCurrentNode(valueNode.node, tracker+1) +func getLineToInsert(fixInfoMetadata *FixInfoMetadata) int { + var lineToInsert int + if fixInfoMetadata.originalListTracker == int(math.Inf(1)) { + lineToInsert = int(math.Inf(1)) } else { - newTracker = skipCurrentNode(currentDFSNode.node, tracker) + lineToInsert = (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker].node.Line - 1 } - - return newTracker + return lineToInsert } -func getNodeLine(dfsOrder *[]NodeInfo, tracker int) int { - if tracker < len(*dfsOrder) { - return (*dfsOrder)[tracker].node.Line +func getNodeLine(nodeList *[]NodeInfo, tracker int) int { + if tracker < len(*nodeList) { + return (*nodeList)[tracker].node.Line } else { return int(math.Inf(1)) } } +// Checks if the node is value node in "key-value" pairs of mapping node +func isValueNodeinMapping(node *NodeInfo) bool { + if node.parent.Kind == yaml.MappingNode && node.index%2 != 0 { + return true + } + return false +} + // Checks if the node is part of single line sequence node and returns the line func isOneLineSequenceNode(list *[]NodeInfo, currentTracker int) (bool, int) { parentNode := (*list)[currentTracker].parent @@ -186,96 +220,7 @@ func isOneLineSequenceNode(list *[]NodeInfo, currentTracker int) (bool, int) { } } -func enocodeIntoYaml(parentNode *yaml.Node, dfsOrder *[]NodeInfo, tracker int) (string, error) { - content := make([]*yaml.Node, 0) - currentNode := (*dfsOrder)[tracker].node - content = append(content, currentNode) - - if parentNode.Kind == yaml.MappingNode { - valueNode := (*dfsOrder)[tracker+1].node - content = append(content, valueNode) - } - - parentForContent := yaml.Node{ - Kind: parentNode.Kind, - Content: content, - } - buf := new(bytes.Buffer) - encoder := yaml.NewEncoder(buf) - encoder.SetIndent(2) - errorEncoding := encoder.Encode(parentForContent) - if errorEncoding != nil { - return "", fmt.Errorf("Error debugging node, %v", errorEncoding.Error()) - } - errorClosingEncoder := encoder.Close() - if errorClosingEncoder != nil { - return "", fmt.Errorf("Error closing encoder: %v", errorClosingEncoder.Error()) - } - return fmt.Sprintf(`%v`, buf.String()), nil -} - -func getContent(parentNode *yaml.Node, dfsOrder *[]NodeInfo, tracker int) string { - content, err := enocodeIntoYaml(parentNode, dfsOrder, tracker) - if err != nil { - logger.L().Fatal("Cannot Encode into YAML") - } - - indentationSpaces := parentNode.Column - 1 - - content = indentContent(content, indentationSpaces) - - return content -} - -func indentContent(content string, indentationSpaces int) string { - indentedContent := "" - indentSpaces := strings.Repeat(" ", indentationSpaces) - - scanner := bufio.NewScanner(strings.NewReader(content)) - for scanner.Scan() { - line := scanner.Text() - indentedContent += (indentSpaces + line + "\n") - } - return indentedContent -} - -func getTracker(list *[]NodeInfo, node *NodeInfo) int { - tracker := 0 - - for !isSameNode((*list)[tracker].node, node.node) { - tracker += 1 - } - - return tracker -} - -// Returns the first node in the given line that is not mapping node -func getFirstNodeInLine(list *[]NodeInfo, line int) int { - tracker := 0 - - currentNode := (*list)[tracker].node - for currentNode.Line != line || currentNode.Kind == yaml.MappingNode { - tracker += 1 - currentNode = (*list)[tracker].node - } - - return tracker -} - -func removeLines(linesToRemove *[]ContentToRemove, linesSlice *[]string) { - for _, lineToRemove := range *linesToRemove { - startLine := lineToRemove.startLine - 1 - endLine := int(math.Min(float64(lineToRemove.endLine), float64(len(*linesSlice)))) - 1 - for line := startLine; line <= endLine; line++ { - lineContent := (*linesSlice)[line] - if isEmptyLineOrComment(lineContent) { - break - } - (*linesSlice)[line] = "*" - } - } -} - +// Checks if nodes are of same kind, value, line and column func isSameNode(nodeOne, nodeTwo *yaml.Node) bool { sameLines := nodeOne.Line == nodeTwo.Line sameColumns := nodeOne.Column == nodeTwo.Column @@ -296,6 +241,124 @@ func isEmptyLineOrComment(lineContent string) bool { return false } +func readDocuments(reader io.Reader, filename string, fileIndex int, decoder yqlib.Decoder) (*list.List, error) { + err := decoder.Init(reader) + if err != nil { + return nil, err + } + inputList := list.New() + var currentIndex uint + + for { + candidateNode, errorReading := decoder.Decode() + + if errors.Is(errorReading, io.EOF) { + switch reader := reader.(type) { + case *os.File: + safelyCloseFile(reader) + } + return inputList, nil + } else if errorReading != nil { + return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading) + } + candidateNode.Document = currentIndex + candidateNode.Filename = filename + candidateNode.FileIndex = fileIndex + candidateNode.EvaluateTogether = true + + inputList.PushBack(candidateNode) + + currentIndex = currentIndex + 1 + } +} + +func safelyCloseFile(file *os.File) { + err := file.Close() + if err != nil { + logger.L().Error("Error Closing File") + } +} + +// Remove the entire line and replace it with the sequence node in fixed info. This way, +// the original formatting is lost. +func replaceSingleLineSequence(fixInfoMetadata *FixInfoMetadata, line int) (int, int) { + originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) + fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) + + currentDFSNode := (*fixInfoMetadata.fixedList)[fixedListTracker] + contentToInsert := constructContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) + + // Remove the Single line + *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, LinesToRemove{ + startLine: line, + endLine: line, + }) + + // Encode entire Sequence Node and Insert + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ + Line: line, + Content: contentToInsert, + }) + + originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) + fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) + + return originalListTracker, fixedListTracker +} + +// Returns the first node in the given line that is not mapping node +func getFirstNodeInLine(list *[]NodeInfo, line int) int { + tracker := 0 + + currentNode := (*list)[tracker].node + for currentNode.Line != line || currentNode.Kind == yaml.MappingNode { + tracker += 1 + currentNode = (*list)[tracker].node + } + + return tracker +} + +// To not mess with the line number while inserting, removed lines are not deleted but replaced with "*" +func removeLines(linesToRemove *[]LinesToRemove, linesSlice *[]string) { + var startLine, endLine int + for _, lineToRemove := range *linesToRemove { + startLine = lineToRemove.startLine - 1 + + if lineToRemove.endLine < len(*linesSlice) { + endLine = lineToRemove.endLine + } else { + // When removing until the end of file and unsure of length of file, endLine is set to Infinity. + // In that case, endLine is read as last line of linesSlice + endLine = len(*linesSlice) - 1 + } + + for line := startLine; line <= endLine; line++ { + lineContent := (*linesSlice)[line] + // When determining the endLine, empty lines and comments which are not intended to be removed are included. + // To deal with that, we need to refrain from removing empty lines and comments + if isEmptyLineOrComment(lineContent) { + break + } + (*linesSlice)[line] = "*" + } + } +} + +// Skips the current node including it's children in DFS order and returns the new tracker. +func skipCurrentNode(node *yaml.Node, currentTracker int) int { + updatedTracker := currentTracker + getChildrenCount(node) + return updatedTracker +} + +func getChildrenCount(node *yaml.Node) int { + totalChildren := 1 + for _, child := range node.Content { + totalChildren += getChildrenCount(child) + } + return totalChildren +} + // Truncates the comments and empty lines at the top of the file and // returns the truncated content func truncateContentAtHead(filePath string) (string, error) { @@ -348,82 +411,18 @@ func truncateContentAtHead(filePath string) (string, error) { return contentAtHead, nil } -func adjustContentLines(contentToAdd *[]ContentToAdd, linesSlice *[]string) { - for contentIdx, content := range *contentToAdd { - line := content.Line +// The current node along with it's children is skipped and the tracker is moved to next sibling +// of current node. If parent is mapping node, "value" in "key-value" pairs is also skipped. +func updateTracker(nodeList *[]NodeInfo, tracker int) int { + currentNode := (*nodeList)[tracker] + var updatedTracker int - // Update Line number to last line if their value is math.Inf - if line == int(math.Inf(1)) { - (*contentToAdd)[contentIdx].Line = len(*linesSlice) - continue - } - - // Adjust line numbers such that there are no "empty lines or comment lines of next nodes" before them - for idx := line - 1; idx >= 0; idx-- { - if isEmptyLineOrComment((*linesSlice)[idx]) { - (*contentToAdd)[contentIdx].Line -= 1 - } else { - break - } - } - } -} - -// Get the lines of existing yaml in a slice -func getLinesSlice(filePath string) ([]string, error) { - lineSlice := make([]string, 0) - - file, err := os.Open(filePath) - if err != nil { - logger.L().Fatal(fmt.Sprintf("Cannot open file %s", filePath)) - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - - for scanner.Scan() { - lineSlice = append(lineSlice, scanner.Text()) - } - if err := scanner.Err(); err != nil { - log.Fatal(err) - return nil, err - } - - return lineSlice, err -} - -// Remove the entire line and replace it with the sequence node in fixed info. This way, -// the original formatting is lost. -func replaceSingleLineSequence(fixInfoMetadata *FixInfoMetadata, line int) (int, int) { - originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) - fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) - - currentDFSNode := (*fixInfoMetadata.fixedList)[fixedListTracker] - content := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) - - // Remove the Single line - *fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{ - startLine: line, - endLine: line, - }) - - // Encode entire Sequence Node and Insert - *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ - Line: line, - Content: content, - }) - - originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) - fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) - - return originalListTracker, fixedListTracker -} - -func writeContentToAdd(writer *bufio.Writer, contentToAdd string) { - scanner := bufio.NewScanner(strings.NewReader(contentToAdd)) - for scanner.Scan() { - line := scanner.Text() - writer.WriteString(line + "\n") + if currentNode.parent.Kind == yaml.MappingNode { + valueNode := (*nodeList)[tracker+1] + updatedTracker = skipCurrentNode(valueNode.node, tracker+1) + } else { + updatedTracker = skipCurrentNode(currentNode.node, tracker) } + + return updatedTracker } From 566b7c29c168fbf6ac9e7fcdd1e5a65a1e558ccc Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Sun, 11 Dec 2022 18:47:52 +0530 Subject: [PATCH 20/39] Simplify dealing with comments and empty lines at the top --- core/pkg/fixhandler/fixhandler.go | 9 +-------- core/pkg/fixhandler/fixhandler_test.go | 2 +- core/pkg/fixhandler/yamlhandler.go | 14 +++++++++----- core/pkg/fixhandler/yamlhelper.go | 18 ++++++++++++++++++ 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index ec00a975..4a38df56 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -217,13 +217,6 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError error) { - // While obtaining fixedYamlNode, comments and empty lines at the top are ignored. In order to deal with that, - // comments and empty lines are removed and they are inserted again when applying fixes to file. - contentAtHead, err := truncateContentAtHead(filePath) - - if err != nil { - logger.L().Fatal("Error truncating comments and empty lines at head") - } originalYamlNode := constructDecodedYaml(filePath) fixedYamlNode := constructFixedYamlNode(filePath, yamlExpression) @@ -232,7 +225,7 @@ func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError e contentToAdd, linesToRemove := getFixInfo(originalList, fixedList) - err = applyFixesToFile(filePath, contentToAdd, linesToRemove, contentAtHead) + err := applyFixesToFile(filePath, contentToAdd, linesToRemove) return err } diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index b8b08028..06cbaab0 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -79,7 +79,7 @@ func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[]string, direct errorMessage := fmt.Sprintf("Content of fixed %s doesn't match content of %s in %s", originalFile, fixedFile, directoryPath) - assert.Equal(t, string(tempFileContent), string(fixedFileContent), errorMessage) + assert.Equal(t, string(fixedFileContent), string(tempFileContent), errorMessage) } } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index d52f4ea6..a129a3f8 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -88,6 +88,12 @@ func matchNodes(nodeOne, nodeTwo *yaml.Node) int { } func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]LinesToRemove) { + + // While obtaining fixedYamlNode, comments and empty lines at the top are ignored. + // This causes a difference in Line numbers across the tree structure. In order to + // counter this, line numbers are adjusted in fixed list. + adjustFixedListLines(originalList, fixedList) + contentToAdd := make([]ContentToAdd, 0) linesToRemove := make([]LinesToRemove, 0) @@ -209,13 +215,14 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { return updatedOriginalTracker, updatedFixedTracker } -func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemove *[]LinesToRemove, contentAtHead string) error { +func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemove *[]LinesToRemove) error { + // Read contents of the file line by line and store in a list linesSlice, err := getLinesSlice(filePath) if err != nil { return err } - + // Clear the current content of file if err := os.Truncate(filePath, 0); err != nil { return err } @@ -237,9 +244,6 @@ func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemo writer := bufio.NewWriter(file) lineIdx, lineToAddIdx := 1, 0 - // Insert the comments and lines at the head removed initially. - writer.WriteString(contentAtHead) - // Ideally, new node is inserted at line before the next node in DFS order. But, when the previous line contains a // comment or empty line, we need to insert new nodes before them. adjustContentLines(contentToAdd, &linesSlice) diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index 502c48a1..85e674fb 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -45,6 +45,24 @@ func adjustContentLines(contentToAdd *[]ContentToAdd, linesSlice *[]string) { } } +func adjustFixedListLines(originalList, fixedList *[]NodeInfo) { + differenceAtTop := (*originalList)[0].node.Line - (*fixedList)[0].node.Line + + if differenceAtTop <= 0 { + return + } + + for _, node := range *fixedList { + // Line numbers should not be changed for new nodes. + if node.node.Line != 0 { + node.node.Line += differenceAtTop + } + } + + return + +} + func constructDFSOrderHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]NodeInfo, index int) { dfsNode := NodeInfo{ node: node, From 926790f49dfbf2740d1670b7573fdb0371f4b240 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Mon, 12 Dec 2022 18:42:59 +0530 Subject: [PATCH 21/39] Dealing with Multiple Resources in a single YAML file --- core/pkg/fixhandler/datastructures.go | 6 +++ core/pkg/fixhandler/fixhandler.go | 53 ++++++++++++++++++++---- core/pkg/fixhandler/fixhandler_test.go | 57 ++++++++++++++++---------- core/pkg/fixhandler/yamlhandler.go | 37 ++++++++++++----- 4 files changed, 113 insertions(+), 40 deletions(-) diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index cb40c060..a3a1572c 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -20,6 +20,12 @@ type ResourceFixInfo struct { YamlExpressions map[string]*armotypes.FixPath Resource *reporthandling.Resource FilePath string + DocumentIndex int +} + +type FileFixInfo struct { + ContentToAdd []ContentToAdd + LinesToRemove []LinesToRemove } // NodeInfo holds extra information about the node diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 4a38df56..5e748e7e 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -145,6 +145,7 @@ func (h *FixHandler) PrepareResourcesToFix() []ResourceFixInfo { FilePath: absolutePath, Resource: resourceObj, YamlExpressions: make(map[string]*armotypes.FixPath, 0), + DocumentIndex: documentIndex, } for i := range result.AssociatedControls { @@ -185,9 +186,22 @@ func (h *FixHandler) PrintExpectedChanges(resourcesToFix []ResourceFixInfo) { func (h *FixHandler) ApplyChanges(resourcesToFix []ResourceFixInfo) (int, []error) { updatedFiles := make(map[string]bool) errors := make([]error, 0) + // Map with key as filepath + filePathFixInfo := make(map[string]*FileFixInfo) for _, resourceToFix := range resourcesToFix { singleExpression := reduceYamlExpressions(&resourceToFix) - if err := h.applyFixToFile(resourceToFix.FilePath, singleExpression); err != nil { + resourceFilePath := resourceToFix.FilePath + + if _, pathExistsInMap := filePathFixInfo[resourceFilePath]; !pathExistsInMap { + contentToAdd := make([]ContentToAdd, 0) + linesToRemove := make([]LinesToRemove, 0) + filePathFixInfo[resourceFilePath] = &FileFixInfo{ + ContentToAdd: contentToAdd, + LinesToRemove: linesToRemove, + } + } + + if err := h.updateFileFixInfo(resourceFilePath, singleExpression, resourceToFix.DocumentIndex, filePathFixInfo[resourceFilePath]); err != nil { errors = append(errors, fmt.Errorf("failed to fix resource [Name: '%s', Kind: '%s'] in '%s': %w ", resourceToFix.Resource.GetName(), @@ -198,6 +212,12 @@ func (h *FixHandler) ApplyChanges(resourcesToFix []ResourceFixInfo) (int, []erro updatedFiles[resourceToFix.FilePath] = true } } + err := h.applyFixToFiles(filePathFixInfo) + + if err != nil { + logger.L().Fatal(fmt.Sprintf("Cannot Apply fixes to files, %v", err.Error())) + errors = append(errors, err) + } return len(updatedFiles), errors } @@ -215,18 +235,35 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str } } -func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError error) { +func (h *FixHandler) updateFileFixInfo(filePath string, yamlExpression string, documentIdx int, fileFixInfo *FileFixInfo) error { + originalYamlNode := (*constructDecodedYaml(filePath))[documentIdx] + fixedYamlNodes, err := constructFixedYamlNodes(filePath, yamlExpression) + if err != nil { + return err + } - originalYamlNode := constructDecodedYaml(filePath) - fixedYamlNode := constructFixedYamlNode(filePath, yamlExpression) + fixedYamlNode := (*fixedYamlNodes)[documentIdx] - originalList := constructDFSOrder(originalYamlNode) - fixedList := constructDFSOrder(fixedYamlNode) + originalList := constructDFSOrder(&originalYamlNode) + fixedList := constructDFSOrder(&fixedYamlNode) contentToAdd, linesToRemove := getFixInfo(originalList, fixedList) - err := applyFixesToFile(filePath, contentToAdd, linesToRemove) - return err + fileFixInfo.ContentToAdd = append(fileFixInfo.ContentToAdd, *contentToAdd...) + fileFixInfo.LinesToRemove = append(fileFixInfo.LinesToRemove, *linesToRemove...) + + return nil + +} + +func (h *FixHandler) applyFixToFiles(filePathFixInfo map[string]*FileFixInfo) error { + for filepath, fixInfo := range filePathFixInfo { + err := applyFixesToFile(filepath, &fixInfo.ContentToAdd, &fixInfo.LinesToRemove) + if err != nil { + return err + } + } + return nil } func (rfi *ResourceFixInfo) addYamlExpressionsFromResourceAssociatedControl(documentIndex int, ac *resourcesresults.ResourceAssociatedControl, skipUserValues bool) { diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 06cbaab0..f1b951aa 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -33,7 +33,7 @@ func fixCommandPath() string { return filepath.Join(filepath.Dir(o), "..", "..", "examples", "fix-command") } -func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[]string, directoryPath string) { +func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[][]string, directoryPath string) { scenarioCount := len(*yamlExpressions) @@ -65,7 +65,20 @@ func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[]string, direct // make changes to temp file h, _ := NewFixHandlerMock() - err = h.applyFixToFile(tempFile.Name(), (*yamlExpressions)[scenario-1]) + + filePathFixInfo := make(map[string]*FileFixInfo) + filePath := tempFile.Name() + filePathFixInfo[filePath] = &FileFixInfo{ + ContentToAdd: make([]ContentToAdd, 0), + LinesToRemove: make([]LinesToRemove, 0), + } + fixInfo := filePathFixInfo[filePath] + + for idx, yamlExpression := range (*yamlExpressions)[scenario-1] { + h.updateFileFixInfo(filePath, yamlExpression, idx, fixInfo) + } + + err = h.applyFixToFiles(filePathFixInfo) assert.NoError(t, err) // Check temp file contents @@ -86,49 +99,49 @@ func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[]string, direct func testDirectoryApplyFix(t *testing.T, directory string) { directoryPath := filepath.Join(fixCommandPath(), directory) - var yamlExpressions []string + var yamlExpressions [][]string switch directory { case "insert_scenarios": - yamlExpressions = []string{ - "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + yamlExpressions = [][]string{ + {"select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false"}, - "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]", + {"select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]"}, - "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", + {"select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]"}, - `select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation |= false | + {`select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation |= false | select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] | select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type |= "RuntimeDefault" | select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false | - select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true`, + select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true`}, - "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + {"select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false"}, - "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", + {"select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]"}, } case "remove_scenarios": - yamlExpressions = []string{ - "del(select(di==0).spec.containers[0].securityContext)", + yamlExpressions = [][]string{ + {"del(select(di==0).spec.containers[0].securityContext)"}, - "del(select(di==0).spec.containers[1])", + {"del(select(di==0).spec.containers[1])"}, - "del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1])", + {"del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1])"}, } case "replace_scenarios": - yamlExpressions = []string{ - "select(di==0).spec.containers[0].securityContext.runAsRoot |= false", + yamlExpressions = [][]string{ + {"select(di==0).spec.containers[0].securityContext.runAsRoot |= false"}, - `select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" | - select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW"`, + {`select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" | + select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW"`}, } case "hybrid_scenarios": - yamlExpressions = []string{ - `del(select(di==0).spec.containers[0].securityContext) | - select(di==0).spec.securityContext.runAsRoot |= false`, + yamlExpressions = [][]string{ + {`del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`}, } } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index a129a3f8..af944f3a 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -4,7 +4,9 @@ import ( "bufio" "bytes" "container/list" + "errors" "fmt" + "io" "io/ioutil" "math" "os" @@ -15,7 +17,7 @@ import ( "gopkg.in/yaml.v3" ) -func constructDecodedYaml(filepath string) *yaml.Node { +func constructDecodedYaml(filepath string) *[]yaml.Node { file, err := ioutil.ReadFile(filepath) if err != nil { logger.L().Fatal("Cannot read file") @@ -23,17 +25,25 @@ func constructDecodedYaml(filepath string) *yaml.Node { fileReader := bytes.NewReader(file) dec := yaml.NewDecoder(fileReader) - var node yaml.Node - err = dec.Decode(&node) + nodes := make([]yaml.Node, 0) + for { + var node yaml.Node + err = dec.Decode(&node) + nodes = append(nodes, node) - if err != nil { - logger.L().Fatal("Cannot Decode Yaml") + // break the loop in case of EOF + if errors.Is(err, io.EOF) { + break + } + if err != nil { + panic(err) + } } - return &node + return &nodes } -func constructFixedYamlNode(filePath, yamlExpression string) *yaml.Node { +func constructFixedYamlNodes(filePath, yamlExpression string) (*[]yaml.Node, error) { preferences := yqlib.ConfiguredYamlPreferences preferences.EvaluateTogether = true decoder := yqlib.NewYamlDecoder(preferences) @@ -41,12 +51,12 @@ func constructFixedYamlNode(filePath, yamlExpression string) *yaml.Node { var allDocuments = list.New() reader, err := constructNewReader(filePath) if err != nil { - return &yaml.Node{} + return nil, err } fileDocuments, err := readDocuments(reader, filePath, 0, decoder) if err != nil { - return &yaml.Node{} + return nil, err } allDocuments.PushBackList(fileDocuments) @@ -58,7 +68,14 @@ func constructFixedYamlNode(filePath, yamlExpression string) *yaml.Node { logger.L().Fatal(fmt.Sprintf("Error fixing YAML, %v", err.Error())) } - return matches.Front().Value.(*yqlib.CandidateNode).Node + fixedNodes := make([]yaml.Node, 0) + var fixedNode *yaml.Node + for match := matches.Front(); match != nil; match = match.Next() { + fixedNode = match.Value.(*yqlib.CandidateNode).Node + fixedNodes = append(fixedNodes, *fixedNode) + } + + return &fixedNodes, nil } func constructDFSOrder(node *yaml.Node) *[]NodeInfo { From 663401d90839c2688db34fc10882f4a20ecc4936 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Mon, 12 Dec 2022 23:09:13 +0530 Subject: [PATCH 22/39] Dealing with Insertion and Removal at last line of a Resource --- core/pkg/fixhandler/datastructures.go | 4 +- core/pkg/fixhandler/yamlhandler.go | 12 ++-- core/pkg/fixhandler/yamlhelper.go | 80 ++++++++++++++++++++------- 3 files changed, 70 insertions(+), 26 deletions(-) diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index a3a1572c..3d4264e2 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -58,6 +58,6 @@ type ContentToAdd struct { // LinesToRemove holds the line numbers to remove from the existing yaml file type LinesToRemove struct { - startLine int - endLine int + StartLine int + EndLine int } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index af944f3a..2dd2864d 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "io/ioutil" - "math" "os" logger "github.com/kubescape/go-logger" @@ -155,7 +154,8 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]LinesT // Some nodes are still not visited if they are inserted at the end of the list for fixedListTracker < len(*fixedList) { - fixInfoMetadata.originalListTracker = int(math.Inf(1)) + // Use negative index of last node in original list as a placeholder to determine the last line number later + fixInfoMetadata.originalListTracker = -(len(*originalList) - 1) fixInfoMetadata.fixedListTracker = fixedListTracker _, fixedListTracker = addLinesToInsert(fixInfoMetadata) } @@ -178,8 +178,8 @@ func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) (int, int) { newOriginalListTracker := updateTracker(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, LinesToRemove{ - startLine: currentDFSNode.node.Line, - endLine: getNodeLine(fixInfoMetadata.originalList, newOriginalListTracker) - 1, + StartLine: currentDFSNode.node.Line, + EndLine: getNodeLine(fixInfoMetadata.originalList, newOriginalListTracker), }) return newOriginalListTracker, fixInfoMetadata.fixedListTracker @@ -239,6 +239,10 @@ func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemo if err != nil { return err } + + // Determining last line required lineSlice. The placeholder for last line is replaced with the real last line + assignLastLine(contentToAdd, linesToRemove, &linesSlice) + // Clear the current content of file if err := os.Truncate(filePath, 0); err != nil { return err diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index 85e674fb..3f7cdfbd 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -28,12 +28,6 @@ func adjustContentLines(contentToAdd *[]ContentToAdd, linesSlice *[]string) { for contentIdx, content := range *contentToAdd { line := content.Line - // Update Line number to last line if their value is math.Inf - if line == int(math.Inf(1)) { - (*contentToAdd)[contentIdx].Line = len(*linesSlice) - continue - } - // Adjust line numbers such that there are no "empty lines or comment lines of next nodes" before them for idx := line - 1; idx >= 0; idx-- { if isEmptyLineOrComment((*linesSlice)[idx]) { @@ -176,19 +170,72 @@ func getLinesSlice(filePath string) ([]string, error) { func getLineToInsert(fixInfoMetadata *FixInfoMetadata) int { var lineToInsert int - if fixInfoMetadata.originalListTracker == int(math.Inf(1)) { - lineToInsert = int(math.Inf(1)) + // Check if lineToInsert is last line + if fixInfoMetadata.originalListTracker < 0 { + originalListTracker := int(math.Abs(float64(fixInfoMetadata.originalListTracker))) + // Storing the negative value of line of last node as a placeholder to determine the last line later. + lineToInsert = -(*fixInfoMetadata.originalList)[originalListTracker].node.Line } else { lineToInsert = (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker].node.Line - 1 } return lineToInsert } +func assignLastLine(contentsToAdd *[]ContentToAdd, linesToRemove *[]LinesToRemove, linesSlice *[]string) { + for idx, contentToAdd := range *contentsToAdd { + if contentToAdd.Line < 0 { + currentLine := int(math.Abs(float64(contentToAdd.Line))) + (*contentsToAdd)[idx].Line, _ = getLastLineOfResource(linesSlice, currentLine) + } + } + + for idx, lineToRemove := range *linesToRemove { + if lineToRemove.EndLine < 0 { + endLine, _ := getLastLineOfResource(linesSlice, lineToRemove.StartLine) + (*linesToRemove)[idx].EndLine = endLine + } + } +} + +func getLastLineOfResource(linesSlice *[]string, currentLine int) (int, error) { + // Get lastlines of all resources... + lastLinesOfResources := make([]int, 0) + for lineNumber, lineContent := range *linesSlice { + if lineContent == "---" { + for lastLine := lineNumber - 1; lastLine >= 0; lastLine-- { + if !isEmptyLineOrComment((*linesSlice)[lastLine]) { + lastLinesOfResources = append(lastLinesOfResources, lastLine+1) + break + } + } + } + } + + lastLine := len(*linesSlice) + for lastLine >= 0 { + if !isEmptyLineOrComment((*linesSlice)[lastLine-1]) { + lastLinesOfResources = append(lastLinesOfResources, lastLine) + break + } else { + lastLine-- + } + } + + // Get last line of the resource we need + for _, endLine := range lastLinesOfResources { + if currentLine <= endLine { + return endLine, nil + } + } + + return 0, fmt.Errorf("Provided line is greater than the length of YAML file") +} + func getNodeLine(nodeList *[]NodeInfo, tracker int) int { if tracker < len(*nodeList) { return (*nodeList)[tracker].node.Line } else { - return int(math.Inf(1)) + return -1 } } @@ -308,8 +355,8 @@ func replaceSingleLineSequence(fixInfoMetadata *FixInfoMetadata, line int) (int, // Remove the Single line *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, LinesToRemove{ - startLine: line, - endLine: line, + StartLine: line, + EndLine: line, }) // Encode entire Sequence Node and Insert @@ -341,15 +388,8 @@ func getFirstNodeInLine(list *[]NodeInfo, line int) int { func removeLines(linesToRemove *[]LinesToRemove, linesSlice *[]string) { var startLine, endLine int for _, lineToRemove := range *linesToRemove { - startLine = lineToRemove.startLine - 1 - - if lineToRemove.endLine < len(*linesSlice) { - endLine = lineToRemove.endLine - } else { - // When removing until the end of file and unsure of length of file, endLine is set to Infinity. - // In that case, endLine is read as last line of linesSlice - endLine = len(*linesSlice) - 1 - } + startLine = lineToRemove.StartLine - 1 + endLine = lineToRemove.EndLine - 1 for line := startLine; line <= endLine; line++ { lineContent := (*linesSlice)[line] From f9a26b7a954343d3f31d29c455180c2fdf2a5bea Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Mon, 12 Dec 2022 23:30:37 +0530 Subject: [PATCH 23/39] Added Test cases for multiple resources in the same YAML file --- core/pkg/fixhandler/fixhandler_test.go | 10 ++++++ .../fixed_yaml_scenario_7.yml | 31 ++++++++++++++++++ .../original_yaml_scenario_7.yml | 27 ++++++++++++++++ .../fixed_yaml_scenario_4.yml | 28 ++++++++++++++++ .../original_yaml_scenario_4.yml | 32 +++++++++++++++++++ 5 files changed, 128 insertions(+) create mode 100644 examples/fix-command/insert_scenarios/fixed_yaml_scenario_7.yml create mode 100644 examples/fix-command/insert_scenarios/original_yaml_scenario_7.yml create mode 100644 examples/fix-command/remove_scenarios/fixed_yaml_scenario_4.yml create mode 100644 examples/fix-command/remove_scenarios/original_yaml_scenario_4.yml diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index f1b951aa..cd0f6b0e 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -119,6 +119,11 @@ func testDirectoryApplyFix(t *testing.T, directory string) { {"select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false"}, {"select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]"}, + + { + "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + "select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + }, } case "remove_scenarios": @@ -128,6 +133,11 @@ func testDirectoryApplyFix(t *testing.T, directory string) { {"del(select(di==0).spec.containers[1])"}, {"del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1])"}, + + { + "del(select(di==0).spec.containers[0].securityContext)", + "del(select(di==1).spec.containers[1])", + }, } case "replace_scenarios": diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_7.yml b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_7.yml new file mode 100644 index 00000000..4085ccbb --- /dev/null +++ b/examples/fix-command/insert_scenarios/fixed_yaml_scenario_7.yml @@ -0,0 +1,31 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + allowPrivilegeEscalation: false + +--- + +# Fix to Apply: +# "select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + allowPrivilegeEscalation: false diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_7.yml b/examples/fix-command/insert_scenarios/original_yaml_scenario_7.yml new file mode 100644 index 00000000..aa6ff7aa --- /dev/null +++ b/examples/fix-command/insert_scenarios/original_yaml_scenario_7.yml @@ -0,0 +1,27 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + +--- + +# Fix to Apply: +# "select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx diff --git a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_4.yml b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_4.yml new file mode 100644 index 00000000..5bfa9988 --- /dev/null +++ b/examples/fix-command/remove_scenarios/fixed_yaml_scenario_4.yml @@ -0,0 +1,28 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + +--- + +# Fix to Apply: +# del(select(di==0).spec.containers[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + diff --git a/examples/fix-command/remove_scenarios/original_yaml_scenario_4.yml b/examples/fix-command/remove_scenarios/original_yaml_scenario_4.yml new file mode 100644 index 00000000..c2aafb03 --- /dev/null +++ b/examples/fix-command/remove_scenarios/original_yaml_scenario_4.yml @@ -0,0 +1,32 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false + +--- + +# Fix to Apply: +# del(select(di==0).spec.containers[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + + - name: container_with_security_issues + image: image_with_security_issues \ No newline at end of file From 8079f9ae7d12117149821ac7209c591e9a7de6f6 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Tue, 13 Dec 2022 15:19:26 +0530 Subject: [PATCH 24/39] Improve Code Readability --- core/pkg/fixhandler/yamlhandler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index 2dd2864d..316b8b14 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -61,7 +61,7 @@ func constructFixedYamlNodes(filePath, yamlExpression string) (*[]yaml.Node, err allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator() - matches, err := allAtOnceEvaluator.EvaluateCandidateNodes(yamlExpression, allDocuments) + fixedCandidateNodes, err := allAtOnceEvaluator.EvaluateCandidateNodes(yamlExpression, allDocuments) if err != nil { logger.L().Fatal(fmt.Sprintf("Error fixing YAML, %v", err.Error())) @@ -69,8 +69,8 @@ func constructFixedYamlNodes(filePath, yamlExpression string) (*[]yaml.Node, err fixedNodes := make([]yaml.Node, 0) var fixedNode *yaml.Node - for match := matches.Front(); match != nil; match = match.Next() { - fixedNode = match.Value.(*yqlib.CandidateNode).Node + for fixedCandidateNode := fixedCandidateNodes.Front(); fixedCandidateNode != nil; fixedCandidateNode = fixedCandidateNode.Next() { + fixedNode = fixedCandidateNode.Value.(*yqlib.CandidateNode).Node fixedNodes = append(fixedNodes, *fixedNode) } From 89fd7eb4394a67644dd4e72dcc3f925bcc3412d8 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Thu, 15 Dec 2022 11:19:04 +0530 Subject: [PATCH 25/39] Move Test files to tesdata directory --- .DS_Store | Bin 0 -> 6148 bytes core/.DS_Store | Bin 0 -> 6148 bytes core/pkg/.DS_Store | Bin 0 -> 6148 bytes core/pkg/fixhandler/.DS_Store | Bin 0 -> 6148 bytes core/pkg/fixhandler/fixhandler_test.go | 8 ++++---- core/pkg/fixhandler/testdata/.DS_Store | Bin 0 -> 6148 bytes .../hybrid_scenarios/fixed_yaml_scenario_1.yml | 0 .../original_yaml_scenario_1.yml | 0 .../insert_scenarios/fixed_yaml_scenario_1.yml | 0 .../insert_scenarios/fixed_yaml_scenario_2.yml | 0 .../insert_scenarios/fixed_yaml_scenario_3.yml | 0 .../insert_scenarios/fixed_yaml_scenario_4.yml | 0 .../insert_scenarios/fixed_yaml_scenario_5.yml | 0 .../insert_scenarios/fixed_yaml_scenario_6.yml | 0 .../insert_scenarios/fixed_yaml_scenario_7.yml | 0 .../original_yaml_scenario_1.yml | 0 .../original_yaml_scenario_2.yml | 0 .../original_yaml_scenario_3.yml | 0 .../original_yaml_scenario_4.yml | 0 .../original_yaml_scenario_5.yml | 0 .../original_yaml_scenario_6.yml | 0 .../original_yaml_scenario_7.yml | 0 .../remove_scenarios/fixed_yaml_scenario_1.yml | 0 .../remove_scenarios/fixed_yaml_scenario_2.yml | 0 .../remove_scenarios/fixed_yaml_scenario_3.yml | 0 .../remove_scenarios/fixed_yaml_scenario_4.yml | 0 .../original_yaml_scenario_1.yml | 0 .../original_yaml_scenario_2.yml | 0 .../original_yaml_scenario_3.yml | 0 .../original_yaml_scenario_4.yml | 0 .../replace_scenarios/fixed_yaml_scenario_1.yml | 0 .../replace_scenarios/fixed_yaml_scenario_2.yml | 0 .../original_yaml_scenario_1.yml | 0 .../original_yaml_scenario_2.yml | 0 34 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 .DS_Store create mode 100644 core/.DS_Store create mode 100644 core/pkg/.DS_Store create mode 100644 core/pkg/fixhandler/.DS_Store create mode 100644 core/pkg/fixhandler/testdata/.DS_Store rename {examples/fix-command => core/pkg/fixhandler/testdata}/hybrid_scenarios/fixed_yaml_scenario_1.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/hybrid_scenarios/original_yaml_scenario_1.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/fixed_yaml_scenario_1.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/fixed_yaml_scenario_2.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/fixed_yaml_scenario_3.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/fixed_yaml_scenario_4.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/fixed_yaml_scenario_5.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/fixed_yaml_scenario_6.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/fixed_yaml_scenario_7.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/original_yaml_scenario_1.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/original_yaml_scenario_2.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/original_yaml_scenario_3.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/original_yaml_scenario_4.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/original_yaml_scenario_5.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/original_yaml_scenario_6.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/insert_scenarios/original_yaml_scenario_7.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/remove_scenarios/fixed_yaml_scenario_1.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/remove_scenarios/fixed_yaml_scenario_2.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/remove_scenarios/fixed_yaml_scenario_3.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/remove_scenarios/fixed_yaml_scenario_4.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/remove_scenarios/original_yaml_scenario_1.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/remove_scenarios/original_yaml_scenario_2.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/remove_scenarios/original_yaml_scenario_3.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/remove_scenarios/original_yaml_scenario_4.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/replace_scenarios/fixed_yaml_scenario_1.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/replace_scenarios/fixed_yaml_scenario_2.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/replace_scenarios/original_yaml_scenario_1.yml (100%) rename {examples/fix-command => core/pkg/fixhandler/testdata}/replace_scenarios/original_yaml_scenario_2.yml (100%) diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..755ea90df766703d4debd8d60124abe511d151b0 GIT binary patch literal 6148 zcmeHK%SyvQ6rE|KO({Yb3SADkE!c+$#Z8Fy2aM=Kr6wk5FlI`cnnfvOtv}?K_&we` zGZBk*D`M}3nRA~rnFE;zV~o4=u+LbVF=j(UbgvE7Ofn+JF@jkSoHP#gJ}}a@?P(q*Xq{RcEfJiE&IlMkcF4~`F!U3*%eynQYJyA`@uyV z6_fVviA-}pPNPf}#9;&}*Oze`%EFcNG@PkgPY3L#-JG;Li^Z@%=!)TBwCsw-Xwd75 z!{O0#*|hie501|!kI7RiUo@c{_*SxGu!2`mKG*Z?WoaVQd+=5HRURQRKnxHA#K7h< zV9o@)wYe0~%83DD;0Fe9e-O|R9fPGtwRJ#;*Jq4dh$x`rTLMuSbPSdnAp*j6DWERp z=83^|IrxRia}1Umbvff|W*EoJTs>a6njQQ?r8Dkmq@EZc238rUYqO2#|0(=3YajWm zC1eo;#K1pefLmj4?82hV+4^mHc-9JNx6n{9uS5j|^p#5h7`TsgR8YqS>X7FcEH&aN S=vU={bP-U5P)7{>0s|jEF-lAT literal 0 HcmV?d00001 diff --git a/core/.DS_Store b/core/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..9b8dc81f1425efded411269adda827c915d29516 GIT binary patch literal 6148 zcmeHK-AcnS6i(c98AIrW!Y%{e4&09j!<$m)3s}($mD$oU%W5;$&RvW_ul0p|5}(I& zk`x^FTEv|L$#;H}=7Z*kF~+@R)MLzHj0tFn9F+<|cV%eLCL?kjBb$eb3c>ma=Qj4& z0l&S*=4{5M;p+SMCvlc{yPtfoR^LBph=yp1NB>!^{5;5)nHMbX(YjG82}?Z)Z{x{o z>>ORHG!No*vd{%_G=Y?dyEu*1%2UfU%5<%70-`CJW2e1d4|@HM9Q22qj$9A>-HtpP zoNqQwaeQ)maXp?TFR6MnbaG%@%b~>*-a)aedG!}*qS7gN%Iq?WkQg8ahyh|?w;8bK zf@tkF4YYbZ6(FH`x*-!37G z7$64z83VjN@<$#l%AT!1mWOApg!Tjt1@j72KtSKR1b~72NLvR@T%rznuEoM2j)Hz& Q4oDXPO$c?wz%MZH1;r~#bpQYW literal 0 HcmV?d00001 diff --git a/core/pkg/.DS_Store b/core/pkg/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e88f67aa7f454ef5d8f95f589d0132da48b502f3 GIT binary patch literal 6148 zcmeHK%}T>S5T0$TO)WwX3Oz1(E!ZCsikDF93mDOZN=-<$!I+i)7@-t$))(?gd>&_Z zw_<5McoDHPF#FBUPnP`>cCr85?=MJ4k3p;gjK2ksaWop&>+d3$FKlcUouX5AuDyXuy|F)@ zj5_}Cl4fU0#lhV6gY&4L_9{EaDjEAx(jV%ADC}d()kTzqD($FA5{`7OX97;iDfKGV z>9o~o)?};Mp4H^E-K^K-LF;ffD>=J+`$wm}`}iSIFNRQo-=3Cji*tBIW98nSykQcn z7PTFeaUqXP%}eWZASkOXbI zOAtznuEoqCMo@$)MKqlf#^>0Tg}tE&y*m1(P6y!{-(TCF%_-3FT!5KTFWitr&Bu d6<1NUpkI=K=vvGSq6dY41T+miFay8Jzy~xsO|}34 literal 0 HcmV?d00001 diff --git a/core/pkg/fixhandler/.DS_Store b/core/pkg/fixhandler/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..b62fad5845ada6533a80fc92c3959271cab320fa GIT binary patch literal 6148 zcmeHK%}T>S5Z<-bZYe?z3Oz1(E!ZCsikDF93mDOZN=-=6V9b^#wTDv3SzpK}@p+ut z-GHSzcoMNQu=~x<&u->}>IF^4fGpdoTpDg@1yt{oeU$n_i{^I6XPV3AEL zCi;se{B{A~9I*`gZ{HszDgfU{Fiqkt?{+`=POZMX*ANZS68GMdT6npi&$E#~yT#F! zQb|ziesCQ}i-~h^uF~9((`cp>;xK}cyPG%-)ncUPX_)C;&p1R=G$&4bxg7NR9XaR^ zR~@+=_PZToV=v!&5+4~cP;xCOLzxmqpnwPmL@8F1Yem~#uE|) z!~iis3~Ubr_6!iM?J1cmB?gFrA2NXZgM^0YS}Y9etphr|KBK>mhypskB@m@W*J5E1 zJRsbp0-98ApBUVvgI(G<*J5GNq%*EphI#DD<>Q6x)xj=xIODEC>WKkjV4Z=Q89I3W zU%)R@`N&^Sp%F1a4E!?&czx`RM^KbKTQ|zXvsOZTfQEu`1u7t*uU!IQfcr>WJB?qW a4souS5T0$TO({YT3Oz1(t=JzCikDF93mDOZN=;1FV49Ww7@-t$))(?gd>&_Z zw_>S1co8WxW%iq$oyjs^%1#ylL}wB<04f0Bpc3XvIQ$^APP!yH>md~S8MmNdfIlCS zrD%5iM+WHKmEjH|c!53m{{F%s9X$oH3NYe5c#5OZxL*GdxqM+`v*;9^vUBY{sMH(# zvPKd%jhFo1lNvP6}nk3;!=Xz$rDLJKH zr8=Fq8qJz)HQTeAoVJ_wnmlM7&SoWNXLtYTwD%Yf67^~*75MMdvSo1&Z)mKj?%5kA zu}bbSVxB$EVq^xG0cK#`8L;P`Q(pH%xjtrq8Td5=bUsK_Lf2wuP#+yQ(B~tq7YIqv zrn>~8wCGyQ4B`rkFsXy8Q2#$!eg9uf;u$l*46GFcqR{oa9W2TA)}`X8*GkkIR1%8I41T4cp<6M= eQY)^aYC*pw1JSjZ8AJ~Xe+XzAcwh$plz~qKg-#g& literal 0 HcmV?d00001 diff --git a/examples/fix-command/hybrid_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/hybrid_scenarios/fixed_yaml_scenario_1.yml similarity index 100% rename from examples/fix-command/hybrid_scenarios/fixed_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/hybrid_scenarios/fixed_yaml_scenario_1.yml diff --git a/examples/fix-command/hybrid_scenarios/original_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/hybrid_scenarios/original_yaml_scenario_1.yml similarity index 100% rename from examples/fix-command/hybrid_scenarios/original_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/hybrid_scenarios/original_yaml_scenario_1.yml diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_1.yml similarity index 100% rename from examples/fix-command/insert_scenarios/fixed_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_1.yml diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_2.yml similarity index 100% rename from examples/fix-command/insert_scenarios/fixed_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_2.yml diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_3.yml similarity index 100% rename from examples/fix-command/insert_scenarios/fixed_yaml_scenario_3.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_3.yml diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_4.yml similarity index 100% rename from examples/fix-command/insert_scenarios/fixed_yaml_scenario_4.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_4.yml diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_5.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_5.yml similarity index 100% rename from examples/fix-command/insert_scenarios/fixed_yaml_scenario_5.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_5.yml diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_6.yml similarity index 100% rename from examples/fix-command/insert_scenarios/fixed_yaml_scenario_6.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_6.yml diff --git a/examples/fix-command/insert_scenarios/fixed_yaml_scenario_7.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_7.yml similarity index 100% rename from examples/fix-command/insert_scenarios/fixed_yaml_scenario_7.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_7.yml diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_1.yml similarity index 100% rename from examples/fix-command/insert_scenarios/original_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_1.yml diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_2.yml similarity index 100% rename from examples/fix-command/insert_scenarios/original_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_2.yml diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_3.yml similarity index 100% rename from examples/fix-command/insert_scenarios/original_yaml_scenario_3.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_3.yml diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_4.yml b/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_4.yml similarity index 100% rename from examples/fix-command/insert_scenarios/original_yaml_scenario_4.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_4.yml diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_5.yml b/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_5.yml similarity index 100% rename from examples/fix-command/insert_scenarios/original_yaml_scenario_5.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_5.yml diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_6.yml b/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_6.yml similarity index 100% rename from examples/fix-command/insert_scenarios/original_yaml_scenario_6.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_6.yml diff --git a/examples/fix-command/insert_scenarios/original_yaml_scenario_7.yml b/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_7.yml similarity index 100% rename from examples/fix-command/insert_scenarios/original_yaml_scenario_7.yml rename to core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_7.yml diff --git a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_1.yml similarity index 100% rename from examples/fix-command/remove_scenarios/fixed_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_1.yml diff --git a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_2.yml similarity index 100% rename from examples/fix-command/remove_scenarios/fixed_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_2.yml diff --git a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_3.yml similarity index 100% rename from examples/fix-command/remove_scenarios/fixed_yaml_scenario_3.yml rename to core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_3.yml diff --git a/examples/fix-command/remove_scenarios/fixed_yaml_scenario_4.yml b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_4.yml similarity index 100% rename from examples/fix-command/remove_scenarios/fixed_yaml_scenario_4.yml rename to core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_4.yml diff --git a/examples/fix-command/remove_scenarios/original_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_1.yml similarity index 100% rename from examples/fix-command/remove_scenarios/original_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_1.yml diff --git a/examples/fix-command/remove_scenarios/original_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_2.yml similarity index 100% rename from examples/fix-command/remove_scenarios/original_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_2.yml diff --git a/examples/fix-command/remove_scenarios/original_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_3.yml similarity index 100% rename from examples/fix-command/remove_scenarios/original_yaml_scenario_3.yml rename to core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_3.yml diff --git a/examples/fix-command/remove_scenarios/original_yaml_scenario_4.yml b/core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_4.yml similarity index 100% rename from examples/fix-command/remove_scenarios/original_yaml_scenario_4.yml rename to core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_4.yml diff --git a/examples/fix-command/replace_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_1.yml similarity index 100% rename from examples/fix-command/replace_scenarios/fixed_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_1.yml diff --git a/examples/fix-command/replace_scenarios/fixed_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_2.yml similarity index 100% rename from examples/fix-command/replace_scenarios/fixed_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_2.yml diff --git a/examples/fix-command/replace_scenarios/original_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/replace_scenarios/original_yaml_scenario_1.yml similarity index 100% rename from examples/fix-command/replace_scenarios/original_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/replace_scenarios/original_yaml_scenario_1.yml diff --git a/examples/fix-command/replace_scenarios/original_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/replace_scenarios/original_yaml_scenario_2.yml similarity index 100% rename from examples/fix-command/replace_scenarios/original_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/replace_scenarios/original_yaml_scenario_2.yml From 218c77f3ae8ecb7c611648f63cf85bc2cc754cfb Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Thu, 15 Dec 2022 11:27:14 +0530 Subject: [PATCH 26/39] Delete .DS_Store files that are added by mistake --- .DS_Store | Bin 6148 -> 0 bytes core/.DS_Store | Bin 6148 -> 0 bytes core/pkg/.DS_Store | Bin 6148 -> 0 bytes core/pkg/fixhandler/.DS_Store | Bin 6148 -> 0 bytes core/pkg/fixhandler/testdata/.DS_Store | Bin 6148 -> 0 bytes 5 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store delete mode 100644 core/.DS_Store delete mode 100644 core/pkg/.DS_Store delete mode 100644 core/pkg/fixhandler/.DS_Store delete mode 100644 core/pkg/fixhandler/testdata/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 755ea90df766703d4debd8d60124abe511d151b0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%SyvQ6rE|KO({Yb3SADkE!c+$#Z8Fy2aM=Kr6wk5FlI`cnnfvOtv}?K_&we` zGZBk*D`M}3nRA~rnFE;zV~o4=u+LbVF=j(UbgvE7Ofn+JF@jkSoHP#gJ}}a@?P(q*Xq{RcEfJiE&IlMkcF4~`F!U3*%eynQYJyA`@uyV z6_fVviA-}pPNPf}#9;&}*Oze`%EFcNG@PkgPY3L#-JG;Li^Z@%=!)TBwCsw-Xwd75 z!{O0#*|hie501|!kI7RiUo@c{_*SxGu!2`mKG*Z?WoaVQd+=5HRURQRKnxHA#K7h< zV9o@)wYe0~%83DD;0Fe9e-O|R9fPGtwRJ#;*Jq4dh$x`rTLMuSbPSdnAp*j6DWERp z=83^|IrxRia}1Umbvff|W*EoJTs>a6njQQ?r8Dkmq@EZc238rUYqO2#|0(=3YajWm zC1eo;#K1pefLmj4?82hV+4^mHc-9JNx6n{9uS5j|^p#5h7`TsgR8YqS>X7FcEH&aN S=vU={bP-U5P)7{>0s|jEF-lAT diff --git a/core/.DS_Store b/core/.DS_Store deleted file mode 100644 index 9b8dc81f1425efded411269adda827c915d29516..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK-AcnS6i(c98AIrW!Y%{e4&09j!<$m)3s}($mD$oU%W5;$&RvW_ul0p|5}(I& zk`x^FTEv|L$#;H}=7Z*kF~+@R)MLzHj0tFn9F+<|cV%eLCL?kjBb$eb3c>ma=Qj4& z0l&S*=4{5M;p+SMCvlc{yPtfoR^LBph=yp1NB>!^{5;5)nHMbX(YjG82}?Z)Z{x{o z>>ORHG!No*vd{%_G=Y?dyEu*1%2UfU%5<%70-`CJW2e1d4|@HM9Q22qj$9A>-HtpP zoNqQwaeQ)maXp?TFR6MnbaG%@%b~>*-a)aedG!}*qS7gN%Iq?WkQg8ahyh|?w;8bK zf@tkF4YYbZ6(FH`x*-!37G z7$64z83VjN@<$#l%AT!1mWOApg!Tjt1@j72KtSKR1b~72NLvR@T%rznuEoM2j)Hz& Q4oDXPO$c?wz%MZH1;r~#bpQYW diff --git a/core/pkg/.DS_Store b/core/pkg/.DS_Store deleted file mode 100644 index e88f67aa7f454ef5d8f95f589d0132da48b502f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5T0$TO)WwX3Oz1(E!ZCsikDF93mDOZN=-<$!I+i)7@-t$))(?gd>&_Z zw_<5McoDHPF#FBUPnP`>cCr85?=MJ4k3p;gjK2ksaWop&>+d3$FKlcUouX5AuDyXuy|F)@ zj5_}Cl4fU0#lhV6gY&4L_9{EaDjEAx(jV%ADC}d()kTzqD($FA5{`7OX97;iDfKGV z>9o~o)?};Mp4H^E-K^K-LF;ffD>=J+`$wm}`}iSIFNRQo-=3Cji*tBIW98nSykQcn z7PTFeaUqXP%}eWZASkOXbI zOAtznuEoqCMo@$)MKqlf#^>0Tg}tE&y*m1(P6y!{-(TCF%_-3FT!5KTFWitr&Bu d6<1NUpkI=K=vvGSq6dY41T+miFay8Jzy~xsO|}34 diff --git a/core/pkg/fixhandler/.DS_Store b/core/pkg/fixhandler/.DS_Store deleted file mode 100644 index b62fad5845ada6533a80fc92c3959271cab320fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z<-bZYe?z3Oz1(E!ZCsikDF93mDOZN=-=6V9b^#wTDv3SzpK}@p+ut z-GHSzcoMNQu=~x<&u->}>IF^4fGpdoTpDg@1yt{oeU$n_i{^I6XPV3AEL zCi;se{B{A~9I*`gZ{HszDgfU{Fiqkt?{+`=POZMX*ANZS68GMdT6npi&$E#~yT#F! zQb|ziesCQ}i-~h^uF~9((`cp>;xK}cyPG%-)ncUPX_)C;&p1R=G$&4bxg7NR9XaR^ zR~@+=_PZToV=v!&5+4~cP;xCOLzxmqpnwPmL@8F1Yem~#uE|) z!~iis3~Ubr_6!iM?J1cmB?gFrA2NXZgM^0YS}Y9etphr|KBK>mhypskB@m@W*J5E1 zJRsbp0-98ApBUVvgI(G<*J5GNq%*EphI#DD<>Q6x)xj=xIODEC>WKkjV4Z=Q89I3W zU%)R@`N&^Sp%F1a4E!?&czx`RM^KbKTQ|zXvsOZTfQEu`1u7t*uU!IQfcr>WJB?qW a4souS5T0$TO({YT3Oz1(t=JzCikDF93mDOZN=;1FV49Ww7@-t$))(?gd>&_Z zw_>S1co8WxW%iq$oyjs^%1#ylL}wB<04f0Bpc3XvIQ$^APP!yH>md~S8MmNdfIlCS zrD%5iM+WHKmEjH|c!53m{{F%s9X$oH3NYe5c#5OZxL*GdxqM+`v*;9^vUBY{sMH(# zvPKd%jhFo1lNvP6}nk3;!=Xz$rDLJKH zr8=Fq8qJz)HQTeAoVJ_wnmlM7&SoWNXLtYTwD%Yf67^~*75MMdvSo1&Z)mKj?%5kA zu}bbSVxB$EVq^xG0cK#`8L;P`Q(pH%xjtrq8Td5=bUsK_Lf2wuP#+yQ(B~tq7YIqv zrn>~8wCGyQ4B`rkFsXy8Q2#$!eg9uf;u$l*46GFcqR{oa9W2TA)}`X8*GkkIR1%8I41T4cp<6M= eQY)^aYC*pw1JSjZ8AJ~Xe+XzAcwh$plz~qKg-#g& From 0d76fffa488fda6780549fafc4e88f5b6abbd3d1 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Thu, 15 Dec 2022 11:54:00 +0530 Subject: [PATCH 27/39] Don't export structs that are not needed outside fixhandler package --- core/pkg/fixhandler/datastructures.go | 30 ++++++------ core/pkg/fixhandler/fixhandler.go | 22 ++++----- core/pkg/fixhandler/fixhandler_test.go | 8 ++-- core/pkg/fixhandler/yamlhandler.go | 36 +++++++------- core/pkg/fixhandler/yamlhelper.go | 66 +++++++++++++------------- 5 files changed, 81 insertions(+), 81 deletions(-) diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index 3d4264e2..d444ac20 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -23,13 +23,13 @@ type ResourceFixInfo struct { DocumentIndex int } -type FileFixInfo struct { - ContentToAdd []ContentToAdd - LinesToRemove []LinesToRemove +type fileFixInfo struct { + contentToAdd []contentToAdd + linesToRemove []linesToRemove } // NodeInfo holds extra information about the node -type NodeInfo struct { +type nodeInfo struct { node *yaml.Node parent *yaml.Node @@ -39,25 +39,25 @@ type NodeInfo struct { // FixInfoMetadata holds the arguments "getFixInfo" function needs to pass to the // functions it uses -type FixInfoMetadata struct { - originalList *[]NodeInfo - fixedList *[]NodeInfo +type fixInfoMetadata struct { + originalList *[]nodeInfo + fixedList *[]nodeInfo originalListTracker int fixedListTracker int - contentToAdd *[]ContentToAdd - linesToRemove *[]LinesToRemove + contentToAdd *[]contentToAdd + linesToRemove *[]linesToRemove } // ContentToAdd holds the information about where to insert the new changes in the existing yaml file -type ContentToAdd struct { +type contentToAdd struct { // Line where the fix should be applied to - Line int + line int // Content is a string representation of the YAML node that describes a suggested fix - Content string + content string } // LinesToRemove holds the line numbers to remove from the existing yaml file -type LinesToRemove struct { - StartLine int - EndLine int +type linesToRemove struct { + startLine int + endLine int } diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 5e748e7e..5fbc41ec 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -187,17 +187,17 @@ func (h *FixHandler) ApplyChanges(resourcesToFix []ResourceFixInfo) (int, []erro updatedFiles := make(map[string]bool) errors := make([]error, 0) // Map with key as filepath - filePathFixInfo := make(map[string]*FileFixInfo) + filePathFixInfo := make(map[string]*fileFixInfo) for _, resourceToFix := range resourcesToFix { singleExpression := reduceYamlExpressions(&resourceToFix) resourceFilePath := resourceToFix.FilePath if _, pathExistsInMap := filePathFixInfo[resourceFilePath]; !pathExistsInMap { - contentToAdd := make([]ContentToAdd, 0) - linesToRemove := make([]LinesToRemove, 0) - filePathFixInfo[resourceFilePath] = &FileFixInfo{ - ContentToAdd: contentToAdd, - LinesToRemove: linesToRemove, + contentToAdd := make([]contentToAdd, 0) + linesToRemove := make([]linesToRemove, 0) + filePathFixInfo[resourceFilePath] = &fileFixInfo{ + contentToAdd: contentToAdd, + linesToRemove: linesToRemove, } } @@ -235,7 +235,7 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str } } -func (h *FixHandler) updateFileFixInfo(filePath string, yamlExpression string, documentIdx int, fileFixInfo *FileFixInfo) error { +func (h *FixHandler) updateFileFixInfo(filePath string, yamlExpression string, documentIdx int, fileFixInfo *fileFixInfo) error { originalYamlNode := (*constructDecodedYaml(filePath))[documentIdx] fixedYamlNodes, err := constructFixedYamlNodes(filePath, yamlExpression) if err != nil { @@ -249,16 +249,16 @@ func (h *FixHandler) updateFileFixInfo(filePath string, yamlExpression string, d contentToAdd, linesToRemove := getFixInfo(originalList, fixedList) - fileFixInfo.ContentToAdd = append(fileFixInfo.ContentToAdd, *contentToAdd...) - fileFixInfo.LinesToRemove = append(fileFixInfo.LinesToRemove, *linesToRemove...) + fileFixInfo.contentToAdd = append(fileFixInfo.contentToAdd, *contentToAdd...) + fileFixInfo.linesToRemove = append(fileFixInfo.linesToRemove, *linesToRemove...) return nil } -func (h *FixHandler) applyFixToFiles(filePathFixInfo map[string]*FileFixInfo) error { +func (h *FixHandler) applyFixToFiles(filePathFixInfo map[string]*fileFixInfo) error { for filepath, fixInfo := range filePathFixInfo { - err := applyFixesToFile(filepath, &fixInfo.ContentToAdd, &fixInfo.LinesToRemove) + err := applyFixesToFile(filepath, &fixInfo.contentToAdd, &fixInfo.linesToRemove) if err != nil { return err } diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 17f27039..59552aea 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -66,11 +66,11 @@ func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[][]string, dire // make changes to temp file h, _ := NewFixHandlerMock() - filePathFixInfo := make(map[string]*FileFixInfo) + filePathFixInfo := make(map[string]*fileFixInfo) filePath := tempFile.Name() - filePathFixInfo[filePath] = &FileFixInfo{ - ContentToAdd: make([]ContentToAdd, 0), - LinesToRemove: make([]LinesToRemove, 0), + filePathFixInfo[filePath] = &fileFixInfo{ + contentToAdd: make([]contentToAdd, 0), + linesToRemove: make([]linesToRemove, 0), } fixInfo := filePathFixInfo[filePath] diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index 316b8b14..39e85832 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -77,8 +77,8 @@ func constructFixedYamlNodes(filePath, yamlExpression string) (*[]yaml.Node, err return &fixedNodes, nil } -func constructDFSOrder(node *yaml.Node) *[]NodeInfo { - dfsOrder := make([]NodeInfo, 0) +func constructDFSOrder(node *yaml.Node) *[]nodeInfo { + dfsOrder := make([]nodeInfo, 0) constructDFSOrderHelper(node, nil, &dfsOrder, 0) return &dfsOrder } @@ -103,19 +103,19 @@ func matchNodes(nodeOne, nodeTwo *yaml.Node) int { } } -func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]LinesToRemove) { +func getFixInfo(originalList, fixedList *[]nodeInfo) (*[]contentToAdd, *[]linesToRemove) { // While obtaining fixedYamlNode, comments and empty lines at the top are ignored. // This causes a difference in Line numbers across the tree structure. In order to // counter this, line numbers are adjusted in fixed list. adjustFixedListLines(originalList, fixedList) - contentToAdd := make([]ContentToAdd, 0) - linesToRemove := make([]LinesToRemove, 0) + contentToAdd := make([]contentToAdd, 0) + linesToRemove := make([]linesToRemove, 0) originalListTracker, fixedListTracker := 0, 0 - fixInfoMetadata := &FixInfoMetadata{ + fixInfoMetadata := &fixInfoMetadata{ originalList: originalList, fixedList: fixedList, originalListTracker: originalListTracker, @@ -165,7 +165,7 @@ func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]LinesT } // Adds the lines to remove and returns the updated originalListTracker -func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) (int, int) { +func addLinesToRemove(fixInfoMetadata *fixInfoMetadata) (int, int) { isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) if isOneLine { @@ -177,16 +177,16 @@ func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) (int, int) { currentDFSNode := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker] newOriginalListTracker := updateTracker(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) - *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, LinesToRemove{ - StartLine: currentDFSNode.node.Line, - EndLine: getNodeLine(fixInfoMetadata.originalList, newOriginalListTracker), + *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, linesToRemove{ + startLine: currentDFSNode.node.Line, + endLine: getNodeLine(fixInfoMetadata.originalList, newOriginalListTracker), }) return newOriginalListTracker, fixInfoMetadata.fixedListTracker } // Adds the lines to insert and returns the updated fixedListTracker -func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { +func addLinesToInsert(fixInfoMetadata *fixInfoMetadata) (int, int) { isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) @@ -201,16 +201,16 @@ func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) (int, int) { newFixedTracker := updateTracker(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) - *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ - Line: lineToInsert, - Content: contentToInsert, + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, contentToAdd{ + line: lineToInsert, + content: contentToInsert, }) return fixInfoMetadata.originalListTracker, newFixedTracker } // Adds the lines to remove and insert and updates the fixedListTracker and originalListTracker -func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { +func updateLinesToReplace(fixInfoMetadata *fixInfoMetadata) (int, int) { isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) @@ -232,7 +232,7 @@ func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) { return updatedOriginalTracker, updatedFixedTracker } -func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemove *[]LinesToRemove) error { +func applyFixesToFile(filePath string, contentToAdd *[]contentToAdd, linesToRemove *[]linesToRemove) error { // Read contents of the file line by line and store in a list linesSlice, err := getLinesSlice(filePath) @@ -270,7 +270,7 @@ func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemo adjustContentLines(contentToAdd, &linesSlice) for lineToAddIdx < len(*contentToAdd) { - for lineIdx <= (*contentToAdd)[lineToAddIdx].Line { + for lineIdx <= (*contentToAdd)[lineToAddIdx].line { // Check if the current line is not removed if linesSlice[lineIdx-1] != "*" { _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") @@ -281,7 +281,7 @@ func applyFixesToFile(filePath string, contentToAdd *[]ContentToAdd, linesToRemo lineIdx += 1 } - content := (*contentToAdd)[lineToAddIdx].Content + content := (*contentToAdd)[lineToAddIdx].content writer.WriteString(content) lineToAddIdx += 1 diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index 3f7cdfbd..e817e503 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -24,14 +24,14 @@ const ( replacedNode ) -func adjustContentLines(contentToAdd *[]ContentToAdd, linesSlice *[]string) { +func adjustContentLines(contentToAdd *[]contentToAdd, linesSlice *[]string) { for contentIdx, content := range *contentToAdd { - line := content.Line + line := content.line // Adjust line numbers such that there are no "empty lines or comment lines of next nodes" before them for idx := line - 1; idx >= 0; idx-- { if isEmptyLineOrComment((*linesSlice)[idx]) { - (*contentToAdd)[contentIdx].Line -= 1 + (*contentToAdd)[contentIdx].line -= 1 } else { break } @@ -39,7 +39,7 @@ func adjustContentLines(contentToAdd *[]ContentToAdd, linesSlice *[]string) { } } -func adjustFixedListLines(originalList, fixedList *[]NodeInfo) { +func adjustFixedListLines(originalList, fixedList *[]nodeInfo) { differenceAtTop := (*originalList)[0].node.Line - (*fixedList)[0].node.Line if differenceAtTop <= 0 { @@ -47,7 +47,7 @@ func adjustFixedListLines(originalList, fixedList *[]NodeInfo) { } for _, node := range *fixedList { - // Line numbers should not be changed for new nodes. + // line numbers should not be changed for new nodes. if node.node.Line != 0 { node.node.Line += differenceAtTop } @@ -57,8 +57,8 @@ func adjustFixedListLines(originalList, fixedList *[]NodeInfo) { } -func constructDFSOrderHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]NodeInfo, index int) { - dfsNode := NodeInfo{ +func constructDFSOrderHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]nodeInfo, index int) { + dfsNode := nodeInfo{ node: node, parent: parent, index: index, @@ -86,7 +86,7 @@ func constructNewReader(filename string) (io.Reader, error) { return reader, nil } -func enocodeIntoYaml(parentNode *yaml.Node, nodeList *[]NodeInfo, tracker int) (string, error) { +func enocodeIntoYaml(parentNode *yaml.Node, nodeList *[]nodeInfo, tracker int) (string, error) { content := make([]*yaml.Node, 0) currentNode := (*nodeList)[tracker].node content = append(content, currentNode) @@ -119,7 +119,7 @@ func enocodeIntoYaml(parentNode *yaml.Node, nodeList *[]NodeInfo, tracker int) ( return fmt.Sprintf(`%v`, buf.String()), nil } -func constructContent(parentNode *yaml.Node, nodeList *[]NodeInfo, tracker int) string { +func constructContent(parentNode *yaml.Node, nodeList *[]nodeInfo, tracker int) string { content, err := enocodeIntoYaml(parentNode, nodeList, tracker) if err != nil { logger.L().Fatal("Cannot Encode into YAML") @@ -168,7 +168,7 @@ func getLinesSlice(filePath string) ([]string, error) { return lineSlice, err } -func getLineToInsert(fixInfoMetadata *FixInfoMetadata) int { +func getLineToInsert(fixInfoMetadata *fixInfoMetadata) int { var lineToInsert int // Check if lineToInsert is last line if fixInfoMetadata.originalListTracker < 0 { @@ -181,18 +181,18 @@ func getLineToInsert(fixInfoMetadata *FixInfoMetadata) int { return lineToInsert } -func assignLastLine(contentsToAdd *[]ContentToAdd, linesToRemove *[]LinesToRemove, linesSlice *[]string) { +func assignLastLine(contentsToAdd *[]contentToAdd, linesToRemove *[]linesToRemove, linesSlice *[]string) { for idx, contentToAdd := range *contentsToAdd { - if contentToAdd.Line < 0 { - currentLine := int(math.Abs(float64(contentToAdd.Line))) - (*contentsToAdd)[idx].Line, _ = getLastLineOfResource(linesSlice, currentLine) + if contentToAdd.line < 0 { + currentLine := int(math.Abs(float64(contentToAdd.line))) + (*contentsToAdd)[idx].line, _ = getLastLineOfResource(linesSlice, currentLine) } } for idx, lineToRemove := range *linesToRemove { - if lineToRemove.EndLine < 0 { - endLine, _ := getLastLineOfResource(linesSlice, lineToRemove.StartLine) - (*linesToRemove)[idx].EndLine = endLine + if lineToRemove.endLine < 0 { + endLine, _ := getLastLineOfResource(linesSlice, lineToRemove.startLine) + (*linesToRemove)[idx].endLine = endLine } } } @@ -231,7 +231,7 @@ func getLastLineOfResource(linesSlice *[]string, currentLine int) (int, error) { return 0, fmt.Errorf("Provided line is greater than the length of YAML file") } -func getNodeLine(nodeList *[]NodeInfo, tracker int) int { +func getNodeLine(nodeList *[]nodeInfo, tracker int) int { if tracker < len(*nodeList) { return (*nodeList)[tracker].node.Line } else { @@ -240,7 +240,7 @@ func getNodeLine(nodeList *[]NodeInfo, tracker int) int { } // Checks if the node is value node in "key-value" pairs of mapping node -func isValueNodeinMapping(node *NodeInfo) bool { +func isValueNodeinMapping(node *nodeInfo) bool { if node.parent.Kind == yaml.MappingNode && node.index%2 != 0 { return true } @@ -248,13 +248,13 @@ func isValueNodeinMapping(node *NodeInfo) bool { } // Checks if the node is part of single line sequence node and returns the line -func isOneLineSequenceNode(list *[]NodeInfo, currentTracker int) (bool, int) { +func isOneLineSequenceNode(list *[]nodeInfo, currentTracker int) (bool, int) { parentNode := (*list)[currentTracker].parent if parentNode.Kind != yaml.SequenceNode { return false, -1 } - var currentNode, prevNode NodeInfo + var currentNode, prevNode nodeInfo currentTracker -= 1 for (*list)[currentTracker].node != parentNode { @@ -346,7 +346,7 @@ func safelyCloseFile(file *os.File) { // Remove the entire line and replace it with the sequence node in fixed info. This way, // the original formatting is lost. -func replaceSingleLineSequence(fixInfoMetadata *FixInfoMetadata, line int) (int, int) { +func replaceSingleLineSequence(fixInfoMetadata *fixInfoMetadata, line int) (int, int) { originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) @@ -354,15 +354,15 @@ func replaceSingleLineSequence(fixInfoMetadata *FixInfoMetadata, line int) (int, contentToInsert := constructContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) // Remove the Single line - *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, LinesToRemove{ - StartLine: line, - EndLine: line, + *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, linesToRemove{ + startLine: line, + endLine: line, }) // Encode entire Sequence Node and Insert - *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{ - Line: line, - Content: contentToInsert, + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, contentToAdd{ + line: line, + content: contentToInsert, }) originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) @@ -372,7 +372,7 @@ func replaceSingleLineSequence(fixInfoMetadata *FixInfoMetadata, line int) (int, } // Returns the first node in the given line that is not mapping node -func getFirstNodeInLine(list *[]NodeInfo, line int) int { +func getFirstNodeInLine(list *[]nodeInfo, line int) int { tracker := 0 currentNode := (*list)[tracker].node @@ -385,11 +385,11 @@ func getFirstNodeInLine(list *[]NodeInfo, line int) int { } // To not mess with the line number while inserting, removed lines are not deleted but replaced with "*" -func removeLines(linesToRemove *[]LinesToRemove, linesSlice *[]string) { +func removeLines(linesToRemove *[]linesToRemove, linesSlice *[]string) { var startLine, endLine int for _, lineToRemove := range *linesToRemove { - startLine = lineToRemove.StartLine - 1 - endLine = lineToRemove.EndLine - 1 + startLine = lineToRemove.startLine - 1 + endLine = lineToRemove.endLine - 1 for line := startLine; line <= endLine; line++ { lineContent := (*linesSlice)[line] @@ -471,7 +471,7 @@ func truncateContentAtHead(filePath string) (string, error) { // The current node along with it's children is skipped and the tracker is moved to next sibling // of current node. If parent is mapping node, "value" in "key-value" pairs is also skipped. -func updateTracker(nodeList *[]NodeInfo, tracker int) int { +func updateTracker(nodeList *[]nodeInfo, tracker int) int { currentNode := (*nodeList)[tracker] var updatedTracker int From e9a8ffbda96f1fab078b9a27ca388631a6977c5c Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Thu, 15 Dec 2022 17:03:54 +0530 Subject: [PATCH 28/39] Add abstraction while adding contentToAdd and linesToBeRemoved to fileFixInfo --- core/pkg/fixhandler/datastructures.go | 18 +++++++++++++----- core/pkg/fixhandler/fixhandler.go | 11 ++++++++--- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index d444ac20..6a60a1ed 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -23,11 +23,6 @@ type ResourceFixInfo struct { DocumentIndex int } -type fileFixInfo struct { - contentToAdd []contentToAdd - linesToRemove []linesToRemove -} - // NodeInfo holds extra information about the node type nodeInfo struct { node *yaml.Node @@ -61,3 +56,16 @@ type linesToRemove struct { startLine int endLine int } + +type fileFixInfo struct { + contentToAdd []contentToAdd + linesToRemove []linesToRemove +} + +func (fileFixInfo *fileFixInfo) addContent(content contentToAdd) { + fileFixInfo.contentToAdd = append(fileFixInfo.contentToAdd, content) +} + +func (fileFixInfo *fileFixInfo) addLinesToRemove(linesToRemove linesToRemove) { + fileFixInfo.linesToRemove = append(fileFixInfo.linesToRemove, linesToRemove) +} diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 5fbc41ec..4be4d682 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -247,10 +247,15 @@ func (h *FixHandler) updateFileFixInfo(filePath string, yamlExpression string, d originalList := constructDFSOrder(&originalYamlNode) fixedList := constructDFSOrder(&fixedYamlNode) - contentToAdd, linesToRemove := getFixInfo(originalList, fixedList) + contentsToAdd, linesToRemove := getFixInfo(originalList, fixedList) - fileFixInfo.contentToAdd = append(fileFixInfo.contentToAdd, *contentToAdd...) - fileFixInfo.linesToRemove = append(fileFixInfo.linesToRemove, *linesToRemove...) + for _, content := range *contentsToAdd { + fileFixInfo.addContent(content) + } + + for _, lines := range *linesToRemove { + fileFixInfo.addLinesToRemove(lines) + } return nil From 38d26960586bdb3e851d4705275ecc08613b68fe Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Thu, 15 Dec 2022 19:14:52 +0530 Subject: [PATCH 29/39] Break updateFileFixInfo function into getResourceFileFix and addResourceFileFix functions --- core/pkg/fixhandler/fixhandler.go | 25 ++++++++++++++----------- core/pkg/fixhandler/fixhandler_test.go | 7 ++++++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 4be4d682..3aabf011 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -193,15 +193,15 @@ func (h *FixHandler) ApplyChanges(resourcesToFix []ResourceFixInfo) (int, []erro resourceFilePath := resourceToFix.FilePath if _, pathExistsInMap := filePathFixInfo[resourceFilePath]; !pathExistsInMap { - contentToAdd := make([]contentToAdd, 0) - linesToRemove := make([]linesToRemove, 0) filePathFixInfo[resourceFilePath] = &fileFixInfo{ - contentToAdd: contentToAdd, - linesToRemove: linesToRemove, + contentToAdd: make([]contentToAdd, 0), + linesToRemove: make([]linesToRemove, 0), } } - if err := h.updateFileFixInfo(resourceFilePath, singleExpression, resourceToFix.DocumentIndex, filePathFixInfo[resourceFilePath]); err != nil { + contentsToAdd, linesToRemove, err := h.getResourceFileFix(resourceFilePath, singleExpression, resourceToFix.DocumentIndex) + + if err != nil { errors = append(errors, fmt.Errorf("failed to fix resource [Name: '%s', Kind: '%s'] in '%s': %w ", resourceToFix.Resource.GetName(), @@ -209,6 +209,7 @@ func (h *FixHandler) ApplyChanges(resourcesToFix []ResourceFixInfo) (int, []erro resourceToFix.FilePath, err)) } else { + h.addResourceFileFix(contentsToAdd, linesToRemove, filePathFixInfo[resourceFilePath]) updatedFiles[resourceToFix.FilePath] = true } } @@ -235,11 +236,11 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str } } -func (h *FixHandler) updateFileFixInfo(filePath string, yamlExpression string, documentIdx int, fileFixInfo *fileFixInfo) error { +func (h *FixHandler) getResourceFileFix(filePath string, yamlExpression string, documentIdx int) (*[]contentToAdd, *[]linesToRemove, error) { originalYamlNode := (*constructDecodedYaml(filePath))[documentIdx] fixedYamlNodes, err := constructFixedYamlNodes(filePath, yamlExpression) if err != nil { - return err + return nil, nil, err } fixedYamlNode := (*fixedYamlNodes)[documentIdx] @@ -249,16 +250,18 @@ func (h *FixHandler) updateFileFixInfo(filePath string, yamlExpression string, d contentsToAdd, linesToRemove := getFixInfo(originalList, fixedList) - for _, content := range *contentsToAdd { + return contentsToAdd, linesToRemove, nil + +} + +func (h *FixHandler) addResourceFileFix(contentToAdd *[]contentToAdd, linesToRemove *[]linesToRemove, fileFixInfo *fileFixInfo) { + for _, content := range *contentToAdd { fileFixInfo.addContent(content) } for _, lines := range *linesToRemove { fileFixInfo.addLinesToRemove(lines) } - - return nil - } func (h *FixHandler) applyFixToFiles(filePathFixInfo map[string]*fileFixInfo) error { diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 59552aea..b7ef9e03 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -75,7 +75,12 @@ func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[][]string, dire fixInfo := filePathFixInfo[filePath] for idx, yamlExpression := range (*yamlExpressions)[scenario-1] { - h.updateFileFixInfo(filePath, yamlExpression, idx, fixInfo) + contentToAdd, linesToRemove, err := h.getResourceFileFix(filePath, yamlExpression, idx) + + if err == nil { + h.addResourceFileFix(contentToAdd, linesToRemove, fixInfo) + } + } err = h.applyFixToFiles(filePathFixInfo) From fa5e7fef23cb194ee415ecc563613bbf18b69f9b Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Fri, 16 Dec 2022 03:25:13 +0530 Subject: [PATCH 30/39] Restructure Code to Improve Code Quality --- core/pkg/fixhandler/fixhandler.go | 117 +++++---- core/pkg/fixhandler/fixhandler_test.go | 247 ++++++++---------- .../fixed_yaml_scenario_1.yml | 2 +- .../fixed_yaml_scenario_1.yml | 2 +- .../fixed_yaml_scenario_2.yml | 2 +- .../fixed_yaml_scenario_3.yml | 2 +- .../fixed_yaml_scenario_5.yml | 2 +- .../fixed_yaml_scenario_6.yml | 2 +- .../fixed_yaml_scenario_1.yml | 2 +- .../fixed_yaml_scenario_2.yml | 1 - .../fixed_yaml_scenario_3.yml | 2 +- .../fixed_yaml_scenario_4.yml | 1 - .../fixed_yaml_scenario_1.yml | 2 +- .../fixed_yaml_scenario_2.yml | 2 +- core/pkg/fixhandler/yamlhandler.go | 113 ++++---- core/pkg/fixhandler/yamlhelper.go | 14 +- 16 files changed, 249 insertions(+), 264 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 3aabf011..7f61d4d5 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -186,39 +186,34 @@ func (h *FixHandler) PrintExpectedChanges(resourcesToFix []ResourceFixInfo) { func (h *FixHandler) ApplyChanges(resourcesToFix []ResourceFixInfo) (int, []error) { updatedFiles := make(map[string]bool) errors := make([]error, 0) - // Map with key as filepath - filePathFixInfo := make(map[string]*fileFixInfo) - for _, resourceToFix := range resourcesToFix { - singleExpression := reduceYamlExpressions(&resourceToFix) - resourceFilePath := resourceToFix.FilePath - if _, pathExistsInMap := filePathFixInfo[resourceFilePath]; !pathExistsInMap { - filePathFixInfo[resourceFilePath] = &fileFixInfo{ - contentToAdd: make([]contentToAdd, 0), - linesToRemove: make([]linesToRemove, 0), - } - } + fileYamlExpressions := h.getFileYamlExpressions(resourcesToFix) - contentsToAdd, linesToRemove, err := h.getResourceFileFix(resourceFilePath, singleExpression, resourceToFix.DocumentIndex) + for filepath, yamlExpression := range fileYamlExpressions { + fileAsString, err := getFileString(filepath) if err != nil { - errors = append(errors, - fmt.Errorf("failed to fix resource [Name: '%s', Kind: '%s'] in '%s': %w ", - resourceToFix.Resource.GetName(), - resourceToFix.Resource.GetKind(), - resourceToFix.FilePath, - err)) + logger.L().Error(err.Error()) + continue + } + + fixedYamlString, err := h.ApplyFix(fileAsString, yamlExpression) + + if err != nil { + errors = append(errors, fmt.Errorf("failed to fix file %s: %w ", filepath, err)) + continue } else { - h.addResourceFileFix(contentsToAdd, linesToRemove, filePathFixInfo[resourceFilePath]) - updatedFiles[resourceToFix.FilePath] = true + updatedFiles[filepath] = true + } + + err = writeFixesToFile(filepath, fixedYamlString) + + if err != nil { + logger.L().Error(fmt.Sprintf("Cannot Apply fixes to file %s, %v", filepath, err.Error())) + errors = append(errors, err) } } - err := h.applyFixToFiles(filePathFixInfo) - if err != nil { - logger.L().Fatal(fmt.Sprintf("Cannot Apply fixes to files, %v", err.Error())) - errors = append(errors, err) - } return len(updatedFiles), errors } @@ -236,42 +231,40 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str } } -func (h *FixHandler) getResourceFileFix(filePath string, yamlExpression string, documentIdx int) (*[]contentToAdd, *[]linesToRemove, error) { - originalYamlNode := (*constructDecodedYaml(filePath))[documentIdx] - fixedYamlNodes, err := constructFixedYamlNodes(filePath, yamlExpression) +func (h *FixHandler) ApplyFix(yamlString, yamlExpression string) (fixedYamlString string, err error) { + yamlLines := strings.Split(yamlString, "\n") + + originalRootNodes := constructDecodedYaml(yamlString) + fixedRootNodes, err := constructFixedYamlNodes(yamlString, yamlExpression) + if err != nil { - return nil, nil, err + return "", err } - fixedYamlNode := (*fixedYamlNodes)[documentIdx] + contentsToAdd, linesToRemove := getFixInfo(originalRootNodes, fixedRootNodes) - originalList := constructDFSOrder(&originalYamlNode) - fixedList := constructDFSOrder(&fixedYamlNode) + fixedYamlLines := getFixedYamlLines(yamlLines, contentsToAdd, linesToRemove) - contentsToAdd, linesToRemove := getFixInfo(originalList, fixedList) - - return contentsToAdd, linesToRemove, nil + fixedYamlString = getFixedYamlString(fixedYamlLines) + return fixedYamlString, nil } -func (h *FixHandler) addResourceFileFix(contentToAdd *[]contentToAdd, linesToRemove *[]linesToRemove, fileFixInfo *fileFixInfo) { - for _, content := range *contentToAdd { - fileFixInfo.addContent(content) - } +func (h *FixHandler) getFileYamlExpressions(resourcesToFix []ResourceFixInfo) map[string]string { + fileYamlExpressions := make(map[string]string, 0) + for _, resourceToFix := range resourcesToFix { + singleExpression := reduceYamlExpressions(&resourceToFix) + resourceFilePath := resourceToFix.FilePath - for _, lines := range *linesToRemove { - fileFixInfo.addLinesToRemove(lines) - } -} - -func (h *FixHandler) applyFixToFiles(filePathFixInfo map[string]*fileFixInfo) error { - for filepath, fixInfo := range filePathFixInfo { - err := applyFixesToFile(filepath, &fixInfo.contentToAdd, &fixInfo.linesToRemove) - if err != nil { - return err + if _, pathExistsInMap := fileYamlExpressions[resourceFilePath]; !pathExistsInMap { + fileYamlExpressions[resourceFilePath] = singleExpression + } else { + fileYamlExpressions[resourceFilePath] = joinStrings(fileYamlExpressions[resourceFilePath], " | ", singleExpression) } + } - return nil + + return fileYamlExpressions } func (rfi *ResourceFixInfo) addYamlExpressionsFromResourceAssociatedControl(documentIndex int, ac *resourcesresults.ResourceAssociatedControl, skipUserValues bool) { @@ -304,6 +297,30 @@ func reduceYamlExpressions(resource *ResourceFixInfo) string { return strings.Join(expressions, " | ") } +func joinStrings(inputStrings ...string) string { + return strings.Join(inputStrings, "") +} + +func getFileString(filepath string) (string, error) { + bytes, err := ioutil.ReadFile(filepath) + + if err != nil { + return "", fmt.Errorf("Error reading file %s", filepath) + } + + return string(bytes), nil +} + +func writeFixesToFile(filepath, content string) error { + err := ioutil.WriteFile(filepath, []byte(content), 0644) + + if err != nil { + return fmt.Errorf("Error writing fixes to file: %w", err) + } + + return nil +} + func fixPathToValidYamlExpression(fixPath, value string, documentIndexInYaml int) string { isStringValue := true if _, err := strconv.ParseBool(value); err == nil { diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index b7ef9e03..8fadad4c 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -1,14 +1,10 @@ package fixhandler import ( - "fmt" - "io/ioutil" "os" "path/filepath" "testing" - "github.com/stretchr/testify/assert" - logger "github.com/kubescape/go-logger" metav1 "github.com/kubescape/kubescape/v2/core/meta/datastructures/v1" reporthandlingv2 "github.com/kubescape/opa-utils/reporthandling/v2" @@ -16,6 +12,12 @@ import ( "gopkg.in/op/go-logging.v1" ) +type indentationTestCase struct { + inputFile string + yamlExpression string + expectedFile string +} + func NewFixHandlerMock() (*FixHandler, error) { backendLoggerLeveled := logging.AddModuleLevel(logging.NewLogBackend(logger.L().GetWriter(), "", 0)) backendLoggerLeveled.SetLevel(logging.ERROR, "") @@ -33,151 +35,128 @@ func getTestdataPath() string { return filepath.Join(currentDir, "testdata") } -func testDirectoryApplyFixHelper(t *testing.T, yamlExpressions *[][]string, directoryPath string) { +func getTestCases() []indentationTestCase { + indentationTestCases := []indentationTestCase{ + // Insertion Scenarios + { + "insert_scenarios/original_yaml_scenario_1.yml", + "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + "insert_scenarios/fixed_yaml_scenario_1.yml", + }, + { + "insert_scenarios/original_yaml_scenario_2.yml", + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]", + "insert_scenarios/fixed_yaml_scenario_2.yml", + }, + { + "insert_scenarios/original_yaml_scenario_3.yml", + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", + "insert_scenarios/fixed_yaml_scenario_3.yml", + }, + { + "insert_scenarios/original_yaml_scenario_4.yml", - scenarioCount := len(*yamlExpressions) + `select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation |= false | + select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] | + select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type |= "RuntimeDefault" | + select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false | + select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true`, - for scenario := 1; scenario <= scenarioCount; scenario++ { - originalFile := fmt.Sprintf("original_yaml_scenario_%d.yml", scenario) - fixedFile := fmt.Sprintf("fixed_yaml_scenario_%d.yml", scenario) + "insert_scenarios/fixed_yaml_scenario_4.yml", + }, + { + "insert_scenarios/original_yaml_scenario_5.yml", + "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + "insert_scenarios/fixed_yaml_scenario_5.yml", + }, + { + "insert_scenarios/original_yaml_scenario_6.yml", + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", + "insert_scenarios/fixed_yaml_scenario_6.yml", + }, + { + "insert_scenarios/original_yaml_scenario_7.yml", - originalFilePath := filepath.Join(directoryPath, originalFile) - fixedFilePath := filepath.Join(directoryPath, fixedFile) + `select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false | + select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false`, - // create temp file - tempFile, err := ioutil.TempFile("", originalFile) - if err != nil { - panic(err) - } - defer os.Remove(tempFile.Name()) + "insert_scenarios/fixed_yaml_scenario_7.yml", + }, - // read original file - originalFileContent, err := ioutil.ReadFile(originalFilePath) - if err != nil { - panic(err) + // Removal Scenarios + + { + "remove_scenarios/original_yaml_scenario_1.yml", + "del(select(di==0).spec.containers[0].securityContext)", + "remove_scenarios/fixed_yaml_scenario_1.yml", + }, + { + "remove_scenarios/original_yaml_scenario_2.yml", + "del(select(di==0).spec.containers[1])", + "remove_scenarios/fixed_yaml_scenario_2.yml", + }, + { + "remove_scenarios/original_yaml_scenario_3.yml", + "del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1])", + "remove_scenarios/fixed_yaml_scenario_3.yml", + }, + { + "remove_scenarios/original_yaml_scenario_4.yml", + `del(select(di==0).spec.containers[0].securityContext) | + del(select(di==1).spec.containers[1])`, + "remove_scenarios/fixed_yaml_scenario_4.yml", + }, + + // Replace Scenarios + { + "replace_scenarios/original_yaml_scenario_1.yml", + "select(di==0).spec.containers[0].securityContext.runAsRoot |= false", + "replace_scenarios/fixed_yaml_scenario_1.yml", + }, + { + "replace_scenarios/original_yaml_scenario_2.yml", + `select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" | + select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW"`, + "replace_scenarios/fixed_yaml_scenario_2.yml", + }, + + // Hybrid Scenarios + { + "hybrid_scenarios/original_yaml_scenario_1.yml", + `del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`, + "hybrid_scenarios/fixed_yaml_scenario_1.yml", + }, + } + + return indentationTestCases +} + +func TestApplyFixKeepsIndentation(t *testing.T) { + testCases := getTestCases() + + for _, tc := range testCases { + getTestDataPath := func(filename string) string { + currentDir, _ := os.Getwd() + currentFile := "testdata/" + filename + return filepath.Join(currentDir, currentFile) } - // write original file contents to temp file - err = ioutil.WriteFile(tempFile.Name(), originalFileContent, 0644) - if err != nil { - panic(err) - } + input, _ := os.ReadFile(getTestDataPath(tc.inputFile)) + want, _ := os.ReadFile(getTestDataPath(tc.expectedFile)) + expression := tc.yamlExpression - // make changes to temp file h, _ := NewFixHandlerMock() - filePathFixInfo := make(map[string]*fileFixInfo) - filePath := tempFile.Name() - filePathFixInfo[filePath] = &fileFixInfo{ - contentToAdd: make([]contentToAdd, 0), - linesToRemove: make([]linesToRemove, 0), + got, _ := h.ApplyFix(string(input), expression) + + if got != string(want) { + t.Errorf("Fixed file does not match the expected.\nGot: <%s>\nWant:<%s>", got, want) } - fixInfo := filePathFixInfo[filePath] - - for idx, yamlExpression := range (*yamlExpressions)[scenario-1] { - contentToAdd, linesToRemove, err := h.getResourceFileFix(filePath, yamlExpression, idx) - - if err == nil { - h.addResourceFileFix(contentToAdd, linesToRemove, fixInfo) - } - - } - - err = h.applyFixToFiles(filePathFixInfo) - assert.NoError(t, err) - - // Check temp file contents - tempFileContent, err := ioutil.ReadFile(tempFile.Name()) - if err != nil { - panic(err) - } - - // Get fixed Yaml file content and check if it is equal to tempFileContent - fixedFileContent, err := ioutil.ReadFile(fixedFilePath) - - errorMessage := fmt.Sprintf("Content of fixed %s doesn't match content of %s in %s", originalFile, fixedFile, directoryPath) - - assert.Equal(t, string(fixedFileContent), string(tempFileContent), errorMessage) } } -func testDirectoryApplyFix(t *testing.T, directory string) { - directoryPath := filepath.Join(getTestdataPath(), directory) - var yamlExpressions [][]string - - switch directory { - case "insert_scenarios": - yamlExpressions = [][]string{ - {"select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false"}, - - {"select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]"}, - - {"select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]"}, - - {`select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation |= false | - select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] | - select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type |= "RuntimeDefault" | - select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false | - select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true`}, - - {"select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false"}, - - {"select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]"}, - - { - "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", - "select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", - }, - } - - case "remove_scenarios": - yamlExpressions = [][]string{ - {"del(select(di==0).spec.containers[0].securityContext)"}, - - {"del(select(di==0).spec.containers[1])"}, - - {"del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1])"}, - - { - "del(select(di==0).spec.containers[0].securityContext)", - "del(select(di==1).spec.containers[1])", - }, - } - - case "replace_scenarios": - yamlExpressions = [][]string{ - {"select(di==0).spec.containers[0].securityContext.runAsRoot |= false"}, - - {`select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" | - select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW"`}, - } - - case "hybrid_scenarios": - yamlExpressions = [][]string{ - {`del(select(di==0).spec.containers[0].securityContext) | - select(di==0).spec.securityContext.runAsRoot |= false`}, - } - } - - testDirectoryApplyFixHelper(t, &yamlExpressions, directoryPath) -} - -func TestFixHandler_applyFixToFile(t *testing.T) { - // Tests for Insert scenarios - testDirectoryApplyFix(t, "insert_scenarios") - - // Tests for Removal scenarios - testDirectoryApplyFix(t, "remove_scenarios") - - // Tests for Replace scenarios - testDirectoryApplyFix(t, "replace_scenarios") - - // Tests for Hybrid Scenarios - testDirectoryApplyFix(t, "hybrid_scenarios") - -} - func Test_fixPathToValidYamlExpression(t *testing.T) { type args struct { fixPath string diff --git a/core/pkg/fixhandler/testdata/hybrid_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/hybrid_scenarios/fixed_yaml_scenario_1.yml index b74c6cdc..5ab9f08c 100644 --- a/core/pkg/fixhandler/testdata/hybrid_scenarios/fixed_yaml_scenario_1.yml +++ b/core/pkg/fixhandler/testdata/hybrid_scenarios/fixed_yaml_scenario_1.yml @@ -16,4 +16,4 @@ spec: - name: nginx_container image: nginx securityContext: - runAsRoot: false + runAsRoot: false \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_1.yml index dbe3c81e..55f3aeb5 100644 --- a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_1.yml +++ b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_1.yml @@ -11,4 +11,4 @@ spec: - name: nginx_container image: nginx securityContext: - allowPrivilegeEscalation: false + allowPrivilegeEscalation: false \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_2.yml index f0d77a9f..ba084424 100644 --- a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_2.yml +++ b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_2.yml @@ -12,4 +12,4 @@ spec: securityContext: capabilities: drop: - - NET_RAW + - NET_RAW \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_3.yml index 60b4f0ed..e673574d 100644 --- a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_3.yml +++ b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_3.yml @@ -13,4 +13,4 @@ spec: capabilities: drop: - NET_RAW - - SYS_ADM + - SYS_ADM \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_5.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_5.yml index 6b66f26b..c9917375 100644 --- a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_5.yml +++ b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_5.yml @@ -15,4 +15,4 @@ spec: # Testing if comments are retained as intended securityContext: - runAsRoot: false + runAsRoot: false \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_6.yml b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_6.yml index b0091e62..d64fecd3 100644 --- a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_6.yml +++ b/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_6.yml @@ -11,4 +11,4 @@ spec: image: nginx securityContext: capabilities: - drop: [NET_RAW, SYS_ADM] + drop: [NET_RAW, SYS_ADM] \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_1.yml index 5c5f896c..8ecc8724 100644 --- a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_1.yml +++ b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_1.yml @@ -9,4 +9,4 @@ metadata: spec: containers: - name: nginx_container - image: nginx + image: nginx \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_2.yml index 43b80fa1..56fc35f8 100644 --- a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_2.yml +++ b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_2.yml @@ -10,4 +10,3 @@ spec: containers: - name: nginx_container image: nginx - diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_3.yml index bc2225bd..b0899e2c 100644 --- a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_3.yml +++ b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_3.yml @@ -11,4 +11,4 @@ spec: image: nginx securityContext: capabilities: - drop: ["NET_RAW"] + drop: ["NET_RAW"] \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_4.yml b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_4.yml index 5bfa9988..fd24bbc9 100644 --- a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_4.yml +++ b/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_4.yml @@ -25,4 +25,3 @@ spec: containers: - name: nginx_container image: nginx - diff --git a/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_1.yml index 067565ca..4a4eda33 100644 --- a/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_1.yml +++ b/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_1.yml @@ -11,4 +11,4 @@ spec: - name: nginx_container image: nginx securityContext: - runAsRoot: false + runAsRoot: false \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_2.yml index 2c8b1da4..4ea9b48c 100644 --- a/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_2.yml +++ b/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_2.yml @@ -15,4 +15,4 @@ spec: capabilities: drop: - "SYS_ADM" - add: ["NET_RAW"] + add: ["NET_RAW"] \ No newline at end of file diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index 39e85832..1c22f24f 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -1,14 +1,11 @@ package fixhandler import ( - "bufio" - "bytes" "container/list" "errors" "fmt" "io" - "io/ioutil" - "os" + "strings" logger "github.com/kubescape/go-logger" "github.com/mikefarah/yq/v4/pkg/yqlib" @@ -16,44 +13,37 @@ import ( "gopkg.in/yaml.v3" ) -func constructDecodedYaml(filepath string) *[]yaml.Node { - file, err := ioutil.ReadFile(filepath) - if err != nil { - logger.L().Fatal("Cannot read file") - } - fileReader := bytes.NewReader(file) +func constructDecodedYaml(yamlString string) *[]yaml.Node { + fileReader := strings.NewReader(yamlString) dec := yaml.NewDecoder(fileReader) nodes := make([]yaml.Node, 0) for { var node yaml.Node - err = dec.Decode(&node) + err := dec.Decode(&node) + nodes = append(nodes, node) - // break the loop in case of EOF if errors.Is(err, io.EOF) { break } if err != nil { - panic(err) + logger.L().Fatal("Cannot decode given document") } } return &nodes } -func constructFixedYamlNodes(filePath, yamlExpression string) (*[]yaml.Node, error) { +func constructFixedYamlNodes(yamlString, yamlExpression string) (*[]yaml.Node, error) { preferences := yqlib.ConfiguredYamlPreferences preferences.EvaluateTogether = true decoder := yqlib.NewYamlDecoder(preferences) var allDocuments = list.New() - reader, err := constructNewReader(filePath) - if err != nil { - return nil, err - } + reader := strings.NewReader(yamlString) - fileDocuments, err := readDocuments(reader, filePath, 0, decoder) + fileDocuments, err := readDocuments(reader, decoder) if err != nil { return nil, err } @@ -103,7 +93,22 @@ func matchNodes(nodeOne, nodeTwo *yaml.Node) int { } } -func getFixInfo(originalList, fixedList *[]nodeInfo) (*[]contentToAdd, *[]linesToRemove) { +func getFixInfo(originalRootNodes, fixedRootNodes *[]yaml.Node) (*[]contentToAdd, *[]linesToRemove) { + contentToAdd := make([]contentToAdd, 0) + linesToRemove := make([]linesToRemove, 0) + + for idx, _ := range *fixedRootNodes { + originalList := constructDFSOrder(&(*originalRootNodes)[idx]) + fixedList := constructDFSOrder(&(*fixedRootNodes)[idx]) + nodeContentToAdd, nodeLinesToRemove := getFixInfoHelper(originalList, fixedList) + contentToAdd = append(contentToAdd, *nodeContentToAdd...) + linesToRemove = append(linesToRemove, *nodeLinesToRemove...) + } + + return &contentToAdd, &linesToRemove +} + +func getFixInfoHelper(originalList, fixedList *[]nodeInfo) (*[]contentToAdd, *[]linesToRemove) { // While obtaining fixedYamlNode, comments and empty lines at the top are ignored. // This causes a difference in Line numbers across the tree structure. In order to @@ -232,71 +237,53 @@ func updateLinesToReplace(fixInfoMetadata *fixInfoMetadata) (int, int) { return updatedOriginalTracker, updatedFixedTracker } -func applyFixesToFile(filePath string, contentToAdd *[]contentToAdd, linesToRemove *[]linesToRemove) error { - // Read contents of the file line by line and store in a list - linesSlice, err := getLinesSlice(filePath) - - if err != nil { - return err - } - - // Determining last line required lineSlice. The placeholder for last line is replaced with the real last line - assignLastLine(contentToAdd, linesToRemove, &linesSlice) - - // Clear the current content of file - if err := os.Truncate(filePath, 0); err != nil { - return err - } - - file, err := os.OpenFile(filePath, os.O_RDWR, 0644) - if err != nil { - return err - } - - defer func() error { - if err := file.Close(); err != nil { - return err +func removeNewLinesAtTheEnd(yamlLines []string) []string { + for idx := 1; idx < len(yamlLines); idx++ { + if yamlLines[len(yamlLines)-idx] != "\n" { + yamlLines = yamlLines[:len(yamlLines)-idx+1] + break } - return nil - }() + } + return yamlLines +} - removeLines(linesToRemove, &linesSlice) +func getFixedYamlLines(yamlLines []string, contentToAdd *[]contentToAdd, linesToRemove *[]linesToRemove) (fixedYamlLines []string) { - writer := bufio.NewWriter(file) + // Determining last line requires original yaml lines slice. The placeholder for last line is replaced with the real last line + assignLastLine(contentToAdd, linesToRemove, &yamlLines) + + removeLines(linesToRemove, &yamlLines) + + fixedYamlLines = make([]string, 0) lineIdx, lineToAddIdx := 1, 0 // Ideally, new node is inserted at line before the next node in DFS order. But, when the previous line contains a // comment or empty line, we need to insert new nodes before them. - adjustContentLines(contentToAdd, &linesSlice) + adjustContentLines(contentToAdd, &yamlLines) for lineToAddIdx < len(*contentToAdd) { for lineIdx <= (*contentToAdd)[lineToAddIdx].line { // Check if the current line is not removed - if linesSlice[lineIdx-1] != "*" { - _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") - if err != nil { - return err - } + if yamlLines[lineIdx-1] != "*" { + fixedYamlLines = append(fixedYamlLines, yamlLines[lineIdx-1]) } lineIdx += 1 } content := (*contentToAdd)[lineToAddIdx].content - writer.WriteString(content) + fixedYamlLines = append(fixedYamlLines, content) lineToAddIdx += 1 } - for lineIdx <= len(linesSlice) { - if linesSlice[lineIdx-1] != "*" { - _, err := writer.WriteString(linesSlice[lineIdx-1] + "\n") - if err != nil { - return err - } + for lineIdx <= len(yamlLines) { + if yamlLines[lineIdx-1] != "*" { + fixedYamlLines = append(fixedYamlLines, yamlLines[lineIdx-1]) } lineIdx += 1 } - writer.Flush() - return nil + fixedYamlLines = removeNewLinesAtTheEnd(fixedYamlLines) + + return fixedYamlLines } diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index e817e503..d566ae9a 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -129,7 +129,7 @@ func constructContent(parentNode *yaml.Node, nodeList *[]nodeInfo, tracker int) content = indentContent(content, indentationSpaces) - return content + return strings.TrimSuffix(content, "\n") } func indentContent(content string, indentationSpaces int) string { @@ -306,12 +306,13 @@ func isEmptyLineOrComment(lineContent string) bool { return false } -func readDocuments(reader io.Reader, filename string, fileIndex int, decoder yqlib.Decoder) (*list.List, error) { +func readDocuments(reader io.Reader, decoder yqlib.Decoder) (*list.List, error) { err := decoder.Init(reader) if err != nil { return nil, err } inputList := list.New() + var currentIndex uint for { @@ -324,11 +325,10 @@ func readDocuments(reader io.Reader, filename string, fileIndex int, decoder yql } return inputList, nil } else if errorReading != nil { - return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading) + return nil, fmt.Errorf("Error Decoding YAML file") } + candidateNode.Document = currentIndex - candidateNode.Filename = filename - candidateNode.FileIndex = fileIndex candidateNode.EvaluateTogether = true inputList.PushBack(candidateNode) @@ -484,3 +484,7 @@ func updateTracker(nodeList *[]nodeInfo, tracker int) int { return updatedTracker } + +func getFixedYamlString(yamlLines []string) (fixedYamlString string) { + return strings.Join(yamlLines, "\n") +} From 76d2154152a22d18bce775976859499f91e03d83 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Fri, 16 Dec 2022 12:56:40 +0530 Subject: [PATCH 31/39] All the minor Changes --- core/pkg/fixhandler/datastructures.go | 13 ------- core/pkg/fixhandler/fixhandler.go | 4 +- core/pkg/fixhandler/yamlhandler.go | 48 +++++++++++------------- core/pkg/fixhandler/yamlhelper.go | 53 ++++++++++++--------------- 4 files changed, 46 insertions(+), 72 deletions(-) diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index 6a60a1ed..211854f4 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -56,16 +56,3 @@ type linesToRemove struct { startLine int endLine int } - -type fileFixInfo struct { - contentToAdd []contentToAdd - linesToRemove []linesToRemove -} - -func (fileFixInfo *fileFixInfo) addContent(content contentToAdd) { - fileFixInfo.contentToAdd = append(fileFixInfo.contentToAdd, content) -} - -func (fileFixInfo *fileFixInfo) addLinesToRemove(linesToRemove linesToRemove) { - fileFixInfo.linesToRemove = append(fileFixInfo.linesToRemove, linesToRemove) -} diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 7f61d4d5..f34005d4 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -234,8 +234,8 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str func (h *FixHandler) ApplyFix(yamlString, yamlExpression string) (fixedYamlString string, err error) { yamlLines := strings.Split(yamlString, "\n") - originalRootNodes := constructDecodedYaml(yamlString) - fixedRootNodes, err := constructFixedYamlNodes(yamlString, yamlExpression) + originalRootNodes := decodeDocumentRoots(yamlString) + fixedRootNodes, err := getFixedNodes(yamlString, yamlExpression) if err != nil { return "", err diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index 1c22f24f..c6977544 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -13,7 +13,8 @@ import ( "gopkg.in/yaml.v3" ) -func constructDecodedYaml(yamlString string) *[]yaml.Node { +// decodeDocumentRoots decodes all YAML documents stored in a given `filepath` and returns a slice of their root nodes +func decodeDocumentRoots(yamlString string) *[]yaml.Node { fileReader := strings.NewReader(yamlString) dec := yaml.NewDecoder(fileReader) @@ -35,7 +36,7 @@ func constructDecodedYaml(yamlString string) *[]yaml.Node { return &nodes } -func constructFixedYamlNodes(yamlString, yamlExpression string) (*[]yaml.Node, error) { +func getFixedNodes(yamlString, yamlExpression string) (*[]yaml.Node, error) { preferences := yqlib.ConfiguredYamlPreferences preferences.EvaluateTogether = true decoder := yqlib.NewYamlDecoder(preferences) @@ -67,29 +68,22 @@ func constructFixedYamlNodes(yamlString, yamlExpression string) (*[]yaml.Node, e return &fixedNodes, nil } -func constructDFSOrder(node *yaml.Node) *[]nodeInfo { +func flattenWithDFS(node *yaml.Node) *[]nodeInfo { dfsOrder := make([]nodeInfo, 0) - constructDFSOrderHelper(node, nil, &dfsOrder, 0) + flattenWithDFSHelper(node, nil, &dfsOrder, 0) return &dfsOrder } -func matchNodes(nodeOne, nodeTwo *yaml.Node) int { +func flattenWithDFSHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]nodeInfo, index int) { + dfsNode := nodeInfo{ + node: node, + parent: parent, + index: index, + } + *dfsOrder = append(*dfsOrder, dfsNode) - isNewNode := nodeTwo.Line == 0 && nodeTwo.Column == 0 - sameLines := nodeOne.Line == nodeTwo.Line - sameColumns := nodeOne.Column == nodeTwo.Column - - isSameNode := isSameNode(nodeOne, nodeTwo) - - switch { - case isSameNode: - return int(sameNodes) - case isNewNode: - return int(insertedNode) - case sameLines && sameColumns: - return int(replacedNode) - default: - return int(removedNode) + for idx, child := range node.Content { + flattenWithDFSHelper(child, node, dfsOrder, idx) } } @@ -97,9 +91,9 @@ func getFixInfo(originalRootNodes, fixedRootNodes *[]yaml.Node) (*[]contentToAdd contentToAdd := make([]contentToAdd, 0) linesToRemove := make([]linesToRemove, 0) - for idx, _ := range *fixedRootNodes { - originalList := constructDFSOrder(&(*originalRootNodes)[idx]) - fixedList := constructDFSOrder(&(*fixedRootNodes)[idx]) + for idx := 0; idx < len(*fixedRootNodes); idx++ { + originalList := flattenWithDFS(&(*originalRootNodes)[idx]) + fixedList := flattenWithDFS(&(*fixedRootNodes)[idx]) nodeContentToAdd, nodeLinesToRemove := getFixInfoHelper(originalList, fixedList) contentToAdd = append(contentToAdd, *nodeContentToAdd...) linesToRemove = append(linesToRemove, *nodeLinesToRemove...) @@ -136,17 +130,17 @@ func getFixInfoHelper(originalList, fixedList *[]nodeInfo) (*[]contentToAdd, *[] fixInfoMetadata.fixedListTracker = fixedListTracker switch matchNodeResult { - case int(sameNodes): + case sameNodes: originalListTracker += 1 fixedListTracker += 1 - case int(removedNode): + case removedNode: originalListTracker, fixedListTracker = addLinesToRemove(fixInfoMetadata) - case int(insertedNode): + case insertedNode: originalListTracker, fixedListTracker = addLinesToInsert(fixInfoMetadata) - case int(replacedNode): + case replacedNode: originalListTracker, fixedListTracker = updateLinesToReplace(fixInfoMetadata) } } diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index d566ae9a..a58ac174 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -17,13 +17,35 @@ import ( "gopkg.in/yaml.v3" ) +type NodeRelation int + const ( - sameNodes = iota + sameNodes NodeRelation = iota insertedNode removedNode replacedNode ) +func matchNodes(nodeOne, nodeTwo *yaml.Node) NodeRelation { + + isNewNode := nodeTwo.Line == 0 && nodeTwo.Column == 0 + sameLines := nodeOne.Line == nodeTwo.Line + sameColumns := nodeOne.Column == nodeTwo.Column + + isSameNode := isSameNode(nodeOne, nodeTwo) + + switch { + case isSameNode: + return sameNodes + case isNewNode: + return insertedNode + case sameLines && sameColumns: + return replacedNode + default: + return removedNode + } +} + func adjustContentLines(contentToAdd *[]contentToAdd, linesSlice *[]string) { for contentIdx, content := range *contentToAdd { line := content.line @@ -57,35 +79,6 @@ func adjustFixedListLines(originalList, fixedList *[]nodeInfo) { } -func constructDFSOrderHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]nodeInfo, index int) { - dfsNode := nodeInfo{ - node: node, - parent: parent, - index: index, - } - *dfsOrder = append(*dfsOrder, dfsNode) - - for idx, child := range node.Content { - constructDFSOrderHelper(child, node, dfsOrder, idx) - } -} - -func constructNewReader(filename string) (io.Reader, error) { - var reader *bufio.Reader - if filename == "-" { - reader = bufio.NewReader(os.Stdin) - } else { - // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, - // and ensuring that it's not possible to give a path to a file outside thar directory. - file, err := os.Open(filename) // #nosec - if err != nil { - return nil, err - } - reader = bufio.NewReader(file) - } - return reader, nil -} - func enocodeIntoYaml(parentNode *yaml.Node, nodeList *[]nodeInfo, tracker int) (string, error) { content := make([]*yaml.Node, 0) currentNode := (*nodeList)[tracker].node From c3771eec7ef5598b3c9ee89f85e6ef52e2ce665f Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Fri, 16 Dec 2022 15:57:23 +0530 Subject: [PATCH 32/39] Remove redundant functions and clean code after refactoring --- core/pkg/fixhandler/fixhandler.go | 40 ++++++------- core/pkg/fixhandler/fixhandler_test.go | 2 +- core/pkg/fixhandler/yamlhandler.go | 2 +- core/pkg/fixhandler/yamlhelper.go | 83 +------------------------- 4 files changed, 25 insertions(+), 102 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index f34005d4..b17ed054 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -245,7 +245,7 @@ func (h *FixHandler) ApplyFix(yamlString, yamlExpression string) (fixedYamlStrin fixedYamlLines := getFixedYamlLines(yamlLines, contentsToAdd, linesToRemove) - fixedYamlString = getFixedYamlString(fixedYamlLines) + fixedYamlString = getStringFromSlice(fixedYamlLines) return fixedYamlString, nil } @@ -297,6 +297,25 @@ func reduceYamlExpressions(resource *ResourceFixInfo) string { return strings.Join(expressions, " | ") } +func fixPathToValidYamlExpression(fixPath, value string, documentIndexInYaml int) string { + isStringValue := true + if _, err := strconv.ParseBool(value); err == nil { + isStringValue = false + } else if _, err := strconv.ParseFloat(value, 64); err == nil { + isStringValue = false + } else if _, err := strconv.Atoi(value); err == nil { + isStringValue = false + } + + // Strings should be quoted + if isStringValue { + value = fmt.Sprintf("\"%s\"", value) + } + + // select document index and add a dot for the root node + return fmt.Sprintf("select(di==%d).%s |= %s", documentIndexInYaml, fixPath, value) +} + func joinStrings(inputStrings ...string) string { return strings.Join(inputStrings, "") } @@ -320,22 +339,3 @@ func writeFixesToFile(filepath, content string) error { return nil } - -func fixPathToValidYamlExpression(fixPath, value string, documentIndexInYaml int) string { - isStringValue := true - if _, err := strconv.ParseBool(value); err == nil { - isStringValue = false - } else if _, err := strconv.ParseFloat(value, 64); err == nil { - isStringValue = false - } else if _, err := strconv.Atoi(value); err == nil { - isStringValue = false - } - - // Strings should be quoted - if isStringValue { - value = fmt.Sprintf("\"%s\"", value) - } - - // select document index and add a dot for the root node - return fmt.Sprintf("select(di==%d).%s |= %s", documentIndexInYaml, fixPath, value) -} diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 8fadad4c..363149c3 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -151,7 +151,7 @@ func TestApplyFixKeepsIndentation(t *testing.T) { got, _ := h.ApplyFix(string(input), expression) if got != string(want) { - t.Errorf("Fixed file does not match the expected.\nGot: <%s>\nWant:<%s>", got, want) + t.Errorf("Fixed file does not match the expected.\n FilePath: %s \nGot: <%s>\nWant:<%s>", tc.inputFile, got, want) } } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index c6977544..1ad233fa 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -196,7 +196,7 @@ func addLinesToInsert(fixInfoMetadata *fixInfoMetadata) (int, int) { currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] lineToInsert := getLineToInsert(fixInfoMetadata) - contentToInsert := constructContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) + contentToInsert := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) newFixedTracker := updateTracker(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index a58ac174..93327b2c 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "log" "math" "os" "strings" @@ -112,7 +111,7 @@ func enocodeIntoYaml(parentNode *yaml.Node, nodeList *[]nodeInfo, tracker int) ( return fmt.Sprintf(`%v`, buf.String()), nil } -func constructContent(parentNode *yaml.Node, nodeList *[]nodeInfo, tracker int) string { +func getContent(parentNode *yaml.Node, nodeList *[]nodeInfo, tracker int) string { content, err := enocodeIntoYaml(parentNode, nodeList, tracker) if err != nil { logger.L().Fatal("Cannot Encode into YAML") @@ -137,30 +136,6 @@ func indentContent(content string, indentationSpaces int) string { return indentedContent } -// Get the lines of existing yaml in a slice -func getLinesSlice(filePath string) ([]string, error) { - lineSlice := make([]string, 0) - - file, err := os.Open(filePath) - if err != nil { - logger.L().Fatal(fmt.Sprintf("Cannot open file %s", filePath)) - return nil, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - - for scanner.Scan() { - lineSlice = append(lineSlice, scanner.Text()) - } - if err := scanner.Err(); err != nil { - log.Fatal(err) - return nil, err - } - - return lineSlice, err -} - func getLineToInsert(fixInfoMetadata *fixInfoMetadata) int { var lineToInsert int // Check if lineToInsert is last line @@ -344,7 +319,7 @@ func replaceSingleLineSequence(fixInfoMetadata *fixInfoMetadata, line int) (int, fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) currentDFSNode := (*fixInfoMetadata.fixedList)[fixedListTracker] - contentToInsert := constructContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) + contentToInsert := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) // Remove the Single line *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, linesToRemove{ @@ -410,58 +385,6 @@ func getChildrenCount(node *yaml.Node) int { return totalChildren } -// Truncates the comments and empty lines at the top of the file and -// returns the truncated content -func truncateContentAtHead(filePath string) (string, error) { - var contentAtHead string - - linesSlice, err := getLinesSlice(filePath) - - if err != nil { - return "", err - } - - if err := os.Truncate(filePath, 0); err != nil { - return "", err - } - - file, err := os.OpenFile(filePath, os.O_RDWR, 0644) - if err != nil { - return "", err - } - - defer func() error { - if err := file.Close(); err != nil { - return err - } - return nil - }() - - lineIdx := 0 - - for lineIdx < len(linesSlice) { - if isEmptyLineOrComment(linesSlice[lineIdx]) { - contentAtHead += (linesSlice[lineIdx] + "\n") - lineIdx += 1 - } else { - break - } - } - - writer := bufio.NewWriter(file) - - for lineIdx < len(linesSlice) { - _, err = writer.WriteString(linesSlice[lineIdx] + "\n") - if err != nil { - return "", err - } - lineIdx += 1 - } - - writer.Flush() - return contentAtHead, nil -} - // The current node along with it's children is skipped and the tracker is moved to next sibling // of current node. If parent is mapping node, "value" in "key-value" pairs is also skipped. func updateTracker(nodeList *[]nodeInfo, tracker int) int { @@ -478,6 +401,6 @@ func updateTracker(nodeList *[]nodeInfo, tracker int) int { return updatedTracker } -func getFixedYamlString(yamlLines []string) (fixedYamlString string) { +func getStringFromSlice(yamlLines []string) (fixedYamlString string) { return strings.Join(yamlLines, "\n") } From 85d2f5c2508fe177c6c66487889ccc817f1a5c58 Mon Sep 17 00:00:00 2001 From: suhasgumma Date: Tue, 27 Dec 2022 13:02:03 +0530 Subject: [PATCH 33/39] Minor Changes to Improve Code Quality --- core/pkg/fixhandler/datastructures.go | 5 ++ core/pkg/fixhandler/fixhandler.go | 29 +++++----- core/pkg/fixhandler/fixhandler_test.go | 4 +- core/pkg/fixhandler/yamlhandler.go | 73 ++++++++++++++------------ core/pkg/fixhandler/yamlhelper.go | 4 +- 5 files changed, 64 insertions(+), 51 deletions(-) diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index 211854f4..c844009c 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -56,3 +56,8 @@ type linesToRemove struct { startLine int endLine int } + +type fileFixInfo struct { + contentsToAdd *[]contentToAdd + linesToRemove *[]linesToRemove +} diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index b17ed054..454bc201 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -193,14 +193,14 @@ func (h *FixHandler) ApplyChanges(resourcesToFix []ResourceFixInfo) (int, []erro fileAsString, err := getFileString(filepath) if err != nil { - logger.L().Error(err.Error()) + errors = append(errors, err) continue } - fixedYamlString, err := h.ApplyFix(fileAsString, yamlExpression) + fixedYamlString, err := h.ApplyFixToContent(fileAsString, yamlExpression) if err != nil { - errors = append(errors, fmt.Errorf("failed to fix file %s: %w ", filepath, err)) + errors = append(errors, fmt.Errorf("Failed to fix file %s: %w ", filepath, err)) continue } else { updatedFiles[filepath] = true @@ -209,7 +209,7 @@ func (h *FixHandler) ApplyChanges(resourcesToFix []ResourceFixInfo) (int, []erro err = writeFixesToFile(filepath, fixedYamlString) if err != nil { - logger.L().Error(fmt.Sprintf("Cannot Apply fixes to file %s, %v", filepath, err.Error())) + logger.L().Error(fmt.Sprintf("Failed to write fixes to file %s, %v", filepath, err.Error())) errors = append(errors, err) } } @@ -231,23 +231,28 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str } } -func (h *FixHandler) ApplyFix(yamlString, yamlExpression string) (fixedYamlString string, err error) { - yamlLines := strings.Split(yamlString, "\n") +func (h *FixHandler) ApplyFixToContent(yamlAsString, yamlExpression string) (fixedString string, err error) { + yamlLines := strings.Split(yamlAsString, "\n") - originalRootNodes := decodeDocumentRoots(yamlString) - fixedRootNodes, err := getFixedNodes(yamlString, yamlExpression) + originalRootNodes, err := decodeDocumentRoots(yamlAsString) if err != nil { return "", err } - contentsToAdd, linesToRemove := getFixInfo(originalRootNodes, fixedRootNodes) + fixedRootNodes, err := getFixedNodes(yamlAsString, yamlExpression) - fixedYamlLines := getFixedYamlLines(yamlLines, contentsToAdd, linesToRemove) + if err != nil { + return "", err + } - fixedYamlString = getStringFromSlice(fixedYamlLines) + fileFixInfo := getFixInfo(originalRootNodes, fixedRootNodes) - return fixedYamlString, nil + fixedYamlLines := getFixedYamlLines(yamlLines, fileFixInfo) + + fixedString = getStringFromSlice(fixedYamlLines) + + return fixedString, nil } func (h *FixHandler) getFileYamlExpressions(resourcesToFix []ResourceFixInfo) map[string]string { diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 363149c3..0323241b 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -148,10 +148,10 @@ func TestApplyFixKeepsIndentation(t *testing.T) { h, _ := NewFixHandlerMock() - got, _ := h.ApplyFix(string(input), expression) + got, _ := h.ApplyFixToContent(string(input), expression) if got != string(want) { - t.Errorf("Fixed file does not match the expected.\n FilePath: %s \nGot: <%s>\nWant:<%s>", tc.inputFile, got, want) + t.Errorf("Fixed file does not match the expected.\n FilePath: %s \n\nGot:\n <%s>\n\n\nWant:\n<%s>", tc.inputFile, got, want) } } diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go index 1ad233fa..b22f78c8 100644 --- a/core/pkg/fixhandler/yamlhandler.go +++ b/core/pkg/fixhandler/yamlhandler.go @@ -7,15 +7,14 @@ import ( "io" "strings" - logger "github.com/kubescape/go-logger" "github.com/mikefarah/yq/v4/pkg/yqlib" "gopkg.in/yaml.v3" ) // decodeDocumentRoots decodes all YAML documents stored in a given `filepath` and returns a slice of their root nodes -func decodeDocumentRoots(yamlString string) *[]yaml.Node { - fileReader := strings.NewReader(yamlString) +func decodeDocumentRoots(yamlAsString string) ([]yaml.Node, error) { + fileReader := strings.NewReader(yamlAsString) dec := yaml.NewDecoder(fileReader) nodes := make([]yaml.Node, 0) @@ -29,20 +28,21 @@ func decodeDocumentRoots(yamlString string) *[]yaml.Node { break } if err != nil { - logger.L().Fatal("Cannot decode given document") + return nil, fmt.Errorf("Cannot Decode File as YAML") + } } - return &nodes + return nodes, nil } -func getFixedNodes(yamlString, yamlExpression string) (*[]yaml.Node, error) { +func getFixedNodes(yamlAsString, yamlExpression string) ([]yaml.Node, error) { preferences := yqlib.ConfiguredYamlPreferences preferences.EvaluateTogether = true decoder := yqlib.NewYamlDecoder(preferences) var allDocuments = list.New() - reader := strings.NewReader(yamlString) + reader := strings.NewReader(yamlAsString) fileDocuments, err := readDocuments(reader, decoder) if err != nil { @@ -55,7 +55,7 @@ func getFixedNodes(yamlString, yamlExpression string) (*[]yaml.Node, error) { fixedCandidateNodes, err := allAtOnceEvaluator.EvaluateCandidateNodes(yamlExpression, allDocuments) if err != nil { - logger.L().Fatal(fmt.Sprintf("Error fixing YAML, %v", err.Error())) + return nil, fmt.Errorf("Error fixing YAML, %w", err) } fixedNodes := make([]yaml.Node, 0) @@ -65,7 +65,7 @@ func getFixedNodes(yamlString, yamlExpression string) (*[]yaml.Node, error) { fixedNodes = append(fixedNodes, *fixedNode) } - return &fixedNodes, nil + return fixedNodes, nil } func flattenWithDFS(node *yaml.Node) *[]nodeInfo { @@ -87,27 +87,30 @@ func flattenWithDFSHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]nodeIn } } -func getFixInfo(originalRootNodes, fixedRootNodes *[]yaml.Node) (*[]contentToAdd, *[]linesToRemove) { +func getFixInfo(originalRootNodes, fixedRootNodes []yaml.Node) fileFixInfo { contentToAdd := make([]contentToAdd, 0) linesToRemove := make([]linesToRemove, 0) - for idx := 0; idx < len(*fixedRootNodes); idx++ { - originalList := flattenWithDFS(&(*originalRootNodes)[idx]) - fixedList := flattenWithDFS(&(*fixedRootNodes)[idx]) - nodeContentToAdd, nodeLinesToRemove := getFixInfoHelper(originalList, fixedList) - contentToAdd = append(contentToAdd, *nodeContentToAdd...) - linesToRemove = append(linesToRemove, *nodeLinesToRemove...) + for idx := 0; idx < len(fixedRootNodes); idx++ { + originalList := flattenWithDFS(&originalRootNodes[idx]) + fixedList := flattenWithDFS(&fixedRootNodes[idx]) + nodeContentToAdd, nodeLinesToRemove := getFixInfoHelper(*originalList, *fixedList) + contentToAdd = append(contentToAdd, nodeContentToAdd...) + linesToRemove = append(linesToRemove, nodeLinesToRemove...) } - return &contentToAdd, &linesToRemove + return fileFixInfo{ + contentsToAdd: &contentToAdd, + linesToRemove: &linesToRemove, + } } -func getFixInfoHelper(originalList, fixedList *[]nodeInfo) (*[]contentToAdd, *[]linesToRemove) { +func getFixInfoHelper(originalList, fixedList []nodeInfo) ([]contentToAdd, []linesToRemove) { // While obtaining fixedYamlNode, comments and empty lines at the top are ignored. // This causes a difference in Line numbers across the tree structure. In order to // counter this, line numbers are adjusted in fixed list. - adjustFixedListLines(originalList, fixedList) + adjustFixedListLines(&originalList, &fixedList) contentToAdd := make([]contentToAdd, 0) linesToRemove := make([]linesToRemove, 0) @@ -115,16 +118,16 @@ func getFixInfoHelper(originalList, fixedList *[]nodeInfo) (*[]contentToAdd, *[] originalListTracker, fixedListTracker := 0, 0 fixInfoMetadata := &fixInfoMetadata{ - originalList: originalList, - fixedList: fixedList, + originalList: &originalList, + fixedList: &fixedList, originalListTracker: originalListTracker, fixedListTracker: fixedListTracker, contentToAdd: &contentToAdd, linesToRemove: &linesToRemove, } - for originalListTracker < len(*originalList) && fixedListTracker < len(*fixedList) { - matchNodeResult := matchNodes((*originalList)[originalListTracker].node, (*fixedList)[fixedListTracker].node) + for originalListTracker < len(originalList) && fixedListTracker < len(fixedList) { + matchNodeResult := matchNodes(originalList[originalListTracker].node, fixedList[fixedListTracker].node) fixInfoMetadata.originalListTracker = originalListTracker fixInfoMetadata.fixedListTracker = fixedListTracker @@ -146,20 +149,20 @@ func getFixInfoHelper(originalList, fixedList *[]nodeInfo) (*[]contentToAdd, *[] } // Some nodes are still not visited if they are removed at the end of the list - for originalListTracker < len(*originalList) { + for originalListTracker < len(originalList) { fixInfoMetadata.originalListTracker = originalListTracker originalListTracker, _ = addLinesToRemove(fixInfoMetadata) } // Some nodes are still not visited if they are inserted at the end of the list - for fixedListTracker < len(*fixedList) { + for fixedListTracker < len(fixedList) { // Use negative index of last node in original list as a placeholder to determine the last line number later - fixInfoMetadata.originalListTracker = -(len(*originalList) - 1) + fixInfoMetadata.originalListTracker = -(len(originalList) - 1) fixInfoMetadata.fixedListTracker = fixedListTracker _, fixedListTracker = addLinesToInsert(fixInfoMetadata) } - return &contentToAdd, &linesToRemove + return contentToAdd, linesToRemove } @@ -169,7 +172,7 @@ func addLinesToRemove(fixInfoMetadata *fixInfoMetadata) (int, int) { if isOneLine { // Remove the entire line and replace it with the sequence node in fixed info. This way, - // the original formatting is lost. + // the original formatting is not lost. return replaceSingleLineSequence(fixInfoMetadata, line) } @@ -241,22 +244,22 @@ func removeNewLinesAtTheEnd(yamlLines []string) []string { return yamlLines } -func getFixedYamlLines(yamlLines []string, contentToAdd *[]contentToAdd, linesToRemove *[]linesToRemove) (fixedYamlLines []string) { +func getFixedYamlLines(yamlLines []string, fileFixInfo fileFixInfo) (fixedYamlLines []string) { // Determining last line requires original yaml lines slice. The placeholder for last line is replaced with the real last line - assignLastLine(contentToAdd, linesToRemove, &yamlLines) + assignLastLine(fileFixInfo.contentsToAdd, fileFixInfo.linesToRemove, &yamlLines) - removeLines(linesToRemove, &yamlLines) + removeLines(fileFixInfo.linesToRemove, &yamlLines) fixedYamlLines = make([]string, 0) lineIdx, lineToAddIdx := 1, 0 // Ideally, new node is inserted at line before the next node in DFS order. But, when the previous line contains a // comment or empty line, we need to insert new nodes before them. - adjustContentLines(contentToAdd, &yamlLines) + adjustContentLines(fileFixInfo.contentsToAdd, &yamlLines) - for lineToAddIdx < len(*contentToAdd) { - for lineIdx <= (*contentToAdd)[lineToAddIdx].line { + for lineToAddIdx < len(*fileFixInfo.contentsToAdd) { + for lineIdx <= (*fileFixInfo.contentsToAdd)[lineToAddIdx].line { // Check if the current line is not removed if yamlLines[lineIdx-1] != "*" { fixedYamlLines = append(fixedYamlLines, yamlLines[lineIdx-1]) @@ -264,7 +267,7 @@ func getFixedYamlLines(yamlLines []string, contentToAdd *[]contentToAdd, linesTo lineIdx += 1 } - content := (*contentToAdd)[lineToAddIdx].content + content := (*fileFixInfo.contentsToAdd)[lineToAddIdx].content fixedYamlLines = append(fixedYamlLines, content) lineToAddIdx += 1 diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go index 93327b2c..ad3267ec 100644 --- a/core/pkg/fixhandler/yamlhelper.go +++ b/core/pkg/fixhandler/yamlhelper.go @@ -277,7 +277,7 @@ func isEmptyLineOrComment(lineContent string) bool { func readDocuments(reader io.Reader, decoder yqlib.Decoder) (*list.List, error) { err := decoder.Init(reader) if err != nil { - return nil, err + return nil, fmt.Errorf("Error Initializing the decoder, %w", err) } inputList := list.New() @@ -293,7 +293,7 @@ func readDocuments(reader io.Reader, decoder yqlib.Decoder) (*list.List, error) } return inputList, nil } else if errorReading != nil { - return nil, fmt.Errorf("Error Decoding YAML file") + return nil, fmt.Errorf("Error Decoding YAML file, %w", errorReading) } candidateNode.Document = currentIndex From 2fce139a9a3af3a48bf3b2deb26cae1db337d31e Mon Sep 17 00:00:00 2001 From: Vlad Klokun Date: Tue, 10 Jan 2023 18:59:39 +0200 Subject: [PATCH 34/39] tests: re-organize autofixing unit tests This change: - Changes test data naming convention to be lexicographically sortable and have input and expected data side-by-side. - Executes each test case in a separate run. --- core/pkg/fixhandler/fixhandler_test.go | 94 ++++++++++--------- .../tc-01-00-input.yaml} | 2 +- .../tc-01-01-expected.yaml} | 2 +- .../tc-01-00-input.yaml} | 2 +- .../tc-01-01-expected.yaml} | 2 +- .../tc-02-00-input.yaml} | 0 .../tc-02-01-expected.yaml} | 0 .../tc-03-00-input.yaml} | 0 .../tc-03-01-expected.yaml} | 0 .../tc-04-00-input.yaml} | 0 .../tc-04-01-expected.yaml} | 0 .../tc-05-00-input.yaml} | 0 .../tc-05-01-expected.yaml} | 0 .../tc-06-00-input.yaml} | 0 .../tc-06-01-expected.yaml} | 0 .../tc-07-00-input.yaml} | 0 .../tc-07-01-expected.yaml} | 0 .../tc-01-00-input.yaml} | 0 .../tc-01-01-expected.yaml} | 0 .../tc-02-00-input.yaml} | 0 .../tc-02-01-expected.yaml} | 0 .../tc-03-00-input.yaml} | 0 .../tc-03-01-expected.yaml} | 0 .../tc-04-00-input.yaml} | 0 .../tc-04-01-expected.yaml} | 0 .../tc-01-00-input.yaml} | 0 .../tc-01-01-expected.yaml} | 0 .../tc-02-00-input.yaml} | 0 .../tc-02-01-expected.yaml} | 0 29 files changed, 56 insertions(+), 46 deletions(-) rename core/pkg/fixhandler/testdata/{hybrid_scenarios/original_yaml_scenario_1.yml => hybrids/tc-01-00-input.yaml} (93%) rename core/pkg/fixhandler/testdata/{hybrid_scenarios/fixed_yaml_scenario_1.yml => hybrids/tc-01-01-expected.yaml} (93%) rename core/pkg/fixhandler/testdata/{insert_scenarios/original_yaml_scenario_1.yml => inserts/tc-01-00-input.yaml} (92%) rename core/pkg/fixhandler/testdata/{insert_scenarios/fixed_yaml_scenario_1.yml => inserts/tc-01-01-expected.yaml} (87%) rename core/pkg/fixhandler/testdata/{insert_scenarios/original_yaml_scenario_2.yml => inserts/tc-02-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/fixed_yaml_scenario_2.yml => inserts/tc-02-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/original_yaml_scenario_3.yml => inserts/tc-03-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/fixed_yaml_scenario_3.yml => inserts/tc-03-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/original_yaml_scenario_4.yml => inserts/tc-04-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/fixed_yaml_scenario_4.yml => inserts/tc-04-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/original_yaml_scenario_5.yml => inserts/tc-05-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/fixed_yaml_scenario_5.yml => inserts/tc-05-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/original_yaml_scenario_6.yml => inserts/tc-06-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/fixed_yaml_scenario_6.yml => inserts/tc-06-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/original_yaml_scenario_7.yml => inserts/tc-07-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{insert_scenarios/fixed_yaml_scenario_7.yml => inserts/tc-07-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{remove_scenarios/original_yaml_scenario_1.yml => removals/tc-01-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{remove_scenarios/fixed_yaml_scenario_1.yml => removals/tc-01-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{remove_scenarios/original_yaml_scenario_2.yml => removals/tc-02-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{remove_scenarios/fixed_yaml_scenario_2.yml => removals/tc-02-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{remove_scenarios/original_yaml_scenario_3.yml => removals/tc-03-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{remove_scenarios/fixed_yaml_scenario_3.yml => removals/tc-03-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{remove_scenarios/original_yaml_scenario_4.yml => removals/tc-04-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{remove_scenarios/fixed_yaml_scenario_4.yml => removals/tc-04-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{replace_scenarios/original_yaml_scenario_1.yml => replaces/tc-01-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{replace_scenarios/fixed_yaml_scenario_1.yml => replaces/tc-01-01-expected.yaml} (100%) rename core/pkg/fixhandler/testdata/{replace_scenarios/original_yaml_scenario_2.yml => replaces/tc-02-00-input.yaml} (100%) rename core/pkg/fixhandler/testdata/{replace_scenarios/fixed_yaml_scenario_2.yml => replaces/tc-02-01-expected.yaml} (100%) diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 0323241b..2cdcd1fe 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -39,22 +39,22 @@ func getTestCases() []indentationTestCase { indentationTestCases := []indentationTestCase{ // Insertion Scenarios { - "insert_scenarios/original_yaml_scenario_1.yml", + "inserts/tc-01-00-input.yaml", "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", - "insert_scenarios/fixed_yaml_scenario_1.yml", + "inserts/tc-01-01-expected.yaml", }, { - "insert_scenarios/original_yaml_scenario_2.yml", + "inserts/tc-02-00-input.yaml", "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]", - "insert_scenarios/fixed_yaml_scenario_2.yml", + "inserts/tc-02-01-expected.yaml", }, { - "insert_scenarios/original_yaml_scenario_3.yml", + "inserts/tc-03-00-input.yaml", "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", - "insert_scenarios/fixed_yaml_scenario_3.yml", + "inserts/tc-03-01-expected.yaml", }, { - "insert_scenarios/original_yaml_scenario_4.yml", + "inserts/tc-04-00-input.yaml", `select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation |= false | select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] | @@ -62,70 +62,69 @@ func getTestCases() []indentationTestCase { select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false | select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true`, - "insert_scenarios/fixed_yaml_scenario_4.yml", + "inserts/tc-04-01-expected.yaml", }, { - "insert_scenarios/original_yaml_scenario_5.yml", + "inserts/tc-05-00-input.yaml", "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", - "insert_scenarios/fixed_yaml_scenario_5.yml", + "inserts/tc-05-01-expected.yaml", }, { - "insert_scenarios/original_yaml_scenario_6.yml", + "inserts/tc-06-00-input.yaml", "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", - "insert_scenarios/fixed_yaml_scenario_6.yml", + "inserts/tc-06-01-expected.yaml", }, { - "insert_scenarios/original_yaml_scenario_7.yml", + "inserts/tc-07-00-input.yaml", `select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false | select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false`, - "insert_scenarios/fixed_yaml_scenario_7.yml", + "inserts/tc-07-01-expected.yaml", }, // Removal Scenarios - { - "remove_scenarios/original_yaml_scenario_1.yml", + "removals/tc-01-00-input.yaml", "del(select(di==0).spec.containers[0].securityContext)", - "remove_scenarios/fixed_yaml_scenario_1.yml", + "removals/tc-01-01-expected.yaml", }, { - "remove_scenarios/original_yaml_scenario_2.yml", + "removals/tc-02-00-input.yaml", "del(select(di==0).spec.containers[1])", - "remove_scenarios/fixed_yaml_scenario_2.yml", + "removals/tc-02-01-expected.yaml", }, { - "remove_scenarios/original_yaml_scenario_3.yml", + "removals/tc-03-00-input.yaml", "del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1])", - "remove_scenarios/fixed_yaml_scenario_3.yml", + "removals/tc-03-01-expected.yaml", }, { - "remove_scenarios/original_yaml_scenario_4.yml", + "removes/tc-04-00-input.yaml", `del(select(di==0).spec.containers[0].securityContext) | del(select(di==1).spec.containers[1])`, - "remove_scenarios/fixed_yaml_scenario_4.yml", + "removes/tc-04-01-expected.yaml", }, // Replace Scenarios { - "replace_scenarios/original_yaml_scenario_1.yml", + "replaces/tc-01-00-input.yaml", "select(di==0).spec.containers[0].securityContext.runAsRoot |= false", - "replace_scenarios/fixed_yaml_scenario_1.yml", + "replaces/tc-01-01-expected.yaml", }, { - "replace_scenarios/original_yaml_scenario_2.yml", + "replaces/tc-02-00-input.yaml", `select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" | select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW"`, - "replace_scenarios/fixed_yaml_scenario_2.yml", + "replaces/tc-02-01-expected.yaml", }, // Hybrid Scenarios { - "hybrid_scenarios/original_yaml_scenario_1.yml", + "hybrids/tc-01-00-input.yaml", `del(select(di==0).spec.containers[0].securityContext) | select(di==0).spec.securityContext.runAsRoot |= false`, - "hybrid_scenarios/fixed_yaml_scenario_1.yml", + "hybrids/tc-01-01-expected.yaml", }, } @@ -136,23 +135,34 @@ func TestApplyFixKeepsIndentation(t *testing.T) { testCases := getTestCases() for _, tc := range testCases { - getTestDataPath := func(filename string) string { - currentDir, _ := os.Getwd() - currentFile := "testdata/" + filename - return filepath.Join(currentDir, currentFile) - } + t.Run(tc.inputFile, func(t *testing.T) { + getTestDataPath := func(filename string) string { + currentDir, _ := os.Getwd() + currentFile := "testdata/" + filename + return filepath.Join(currentDir, currentFile) + } - input, _ := os.ReadFile(getTestDataPath(tc.inputFile)) - want, _ := os.ReadFile(getTestDataPath(tc.expectedFile)) - expression := tc.yamlExpression + input, _ := os.ReadFile(getTestDataPath(tc.inputFile)) + want, _ := os.ReadFile(getTestDataPath(tc.expectedFile)) + expression := tc.yamlExpression - h, _ := NewFixHandlerMock() + h, _ := NewFixHandlerMock() - got, _ := h.ApplyFixToContent(string(input), expression) + got, _ := h.ApplyFixToContent(string(input), expression) - if got != string(want) { - t.Errorf("Fixed file does not match the expected.\n FilePath: %s \n\nGot:\n <%s>\n\n\nWant:\n<%s>", tc.inputFile, got, want) - } + if got != string(want) { + t.Errorf( + "Contents of the fixed file don't match the expectation.\n"+ + "FilePath: %s\n\n"+ + "Got:\n<%s>\n\n"+ + "Want:\n<%s>", + tc.inputFile, + got, + want, + ) + } + }, + ) } } diff --git a/core/pkg/fixhandler/testdata/hybrid_scenarios/original_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/hybrids/tc-01-00-input.yaml similarity index 93% rename from core/pkg/fixhandler/testdata/hybrid_scenarios/original_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/hybrids/tc-01-00-input.yaml index c2afc6bd..f8f5b61a 100644 --- a/core/pkg/fixhandler/testdata/hybrid_scenarios/original_yaml_scenario_1.yml +++ b/core/pkg/fixhandler/testdata/hybrids/tc-01-00-input.yaml @@ -16,4 +16,4 @@ spec: - name: nginx_container image: nginx securityContext: - runAsRoot: true \ No newline at end of file + runAsRoot: true diff --git a/core/pkg/fixhandler/testdata/hybrid_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/hybrids/tc-01-01-expected.yaml similarity index 93% rename from core/pkg/fixhandler/testdata/hybrid_scenarios/fixed_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/hybrids/tc-01-01-expected.yaml index 5ab9f08c..b74c6cdc 100644 --- a/core/pkg/fixhandler/testdata/hybrid_scenarios/fixed_yaml_scenario_1.yml +++ b/core/pkg/fixhandler/testdata/hybrids/tc-01-01-expected.yaml @@ -16,4 +16,4 @@ spec: - name: nginx_container image: nginx securityContext: - runAsRoot: false \ No newline at end of file + runAsRoot: false diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/inserts/tc-01-00-input.yaml similarity index 92% rename from core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/inserts/tc-01-00-input.yaml index 7c15eec3..c6be2de9 100644 --- a/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_1.yml +++ b/core/pkg/fixhandler/testdata/inserts/tc-01-00-input.yaml @@ -9,4 +9,4 @@ metadata: spec: containers: - name: nginx_container - image: nginx \ No newline at end of file + image: nginx diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/inserts/tc-01-01-expected.yaml similarity index 87% rename from core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/inserts/tc-01-01-expected.yaml index 55f3aeb5..dbe3c81e 100644 --- a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_1.yml +++ b/core/pkg/fixhandler/testdata/inserts/tc-01-01-expected.yaml @@ -11,4 +11,4 @@ spec: - name: nginx_container image: nginx securityContext: - allowPrivilegeEscalation: false \ No newline at end of file + allowPrivilegeEscalation: false diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/inserts/tc-02-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/inserts/tc-02-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/inserts/tc-02-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/inserts/tc-02-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/inserts/tc-03-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_3.yml rename to core/pkg/fixhandler/testdata/inserts/tc-03-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/inserts/tc-03-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_3.yml rename to core/pkg/fixhandler/testdata/inserts/tc-03-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_4.yml b/core/pkg/fixhandler/testdata/inserts/tc-04-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_4.yml rename to core/pkg/fixhandler/testdata/inserts/tc-04-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_4.yml b/core/pkg/fixhandler/testdata/inserts/tc-04-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_4.yml rename to core/pkg/fixhandler/testdata/inserts/tc-04-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_5.yml b/core/pkg/fixhandler/testdata/inserts/tc-05-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_5.yml rename to core/pkg/fixhandler/testdata/inserts/tc-05-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_5.yml b/core/pkg/fixhandler/testdata/inserts/tc-05-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_5.yml rename to core/pkg/fixhandler/testdata/inserts/tc-05-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_6.yml b/core/pkg/fixhandler/testdata/inserts/tc-06-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_6.yml rename to core/pkg/fixhandler/testdata/inserts/tc-06-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_6.yml b/core/pkg/fixhandler/testdata/inserts/tc-06-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_6.yml rename to core/pkg/fixhandler/testdata/inserts/tc-06-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_7.yml b/core/pkg/fixhandler/testdata/inserts/tc-07-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/original_yaml_scenario_7.yml rename to core/pkg/fixhandler/testdata/inserts/tc-07-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_7.yml b/core/pkg/fixhandler/testdata/inserts/tc-07-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/insert_scenarios/fixed_yaml_scenario_7.yml rename to core/pkg/fixhandler/testdata/inserts/tc-07-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/removals/tc-01-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/removals/tc-01-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/removals/tc-01-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/removals/tc-01-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/removals/tc-02-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/removals/tc-02-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/removals/tc-02-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/removals/tc-02-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/removals/tc-03-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_3.yml rename to core/pkg/fixhandler/testdata/removals/tc-03-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_3.yml b/core/pkg/fixhandler/testdata/removals/tc-03-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_3.yml rename to core/pkg/fixhandler/testdata/removals/tc-03-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_4.yml b/core/pkg/fixhandler/testdata/removals/tc-04-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/remove_scenarios/original_yaml_scenario_4.yml rename to core/pkg/fixhandler/testdata/removals/tc-04-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_4.yml b/core/pkg/fixhandler/testdata/removals/tc-04-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/remove_scenarios/fixed_yaml_scenario_4.yml rename to core/pkg/fixhandler/testdata/removals/tc-04-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/replace_scenarios/original_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/replaces/tc-01-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/replace_scenarios/original_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/replaces/tc-01-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_1.yml b/core/pkg/fixhandler/testdata/replaces/tc-01-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_1.yml rename to core/pkg/fixhandler/testdata/replaces/tc-01-01-expected.yaml diff --git a/core/pkg/fixhandler/testdata/replace_scenarios/original_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/replaces/tc-02-00-input.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/replace_scenarios/original_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/replaces/tc-02-00-input.yaml diff --git a/core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_2.yml b/core/pkg/fixhandler/testdata/replaces/tc-02-01-expected.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/replace_scenarios/fixed_yaml_scenario_2.yml rename to core/pkg/fixhandler/testdata/replaces/tc-02-01-expected.yaml From da3bc8e8ea88b6675c97ae1475e0c9810a43bd07 Mon Sep 17 00:00:00 2001 From: Vlad Klokun Date: Tue, 10 Jan 2023 19:04:07 +0200 Subject: [PATCH 35/39] tests: test autofixing indented lists in hybrid scenarios --- core/pkg/fixhandler/fixhandler_test.go | 6 ++++++ .../hybrids/tc-02-00-input-indented-list.yaml | 19 +++++++++++++++++++ .../testdata/hybrids/tc-02-01-expected.yaml | 19 +++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 core/pkg/fixhandler/testdata/hybrids/tc-02-00-input-indented-list.yaml create mode 100644 core/pkg/fixhandler/testdata/hybrids/tc-02-01-expected.yaml diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 2cdcd1fe..6bdd293e 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -126,6 +126,12 @@ func getTestCases() []indentationTestCase { select(di==0).spec.securityContext.runAsRoot |= false`, "hybrids/tc-01-01-expected.yaml", }, + { + "hybrids/tc-02-00-input-indented-list.yaml", + `del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`, + "hybrids/tc-02-01-expected.yaml", + }, } return indentationTestCases diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-02-00-input-indented-list.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-02-00-input-indented-list.yaml new file mode 100644 index 00000000..8967d5f6 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-02-00-input-indented-list.yaml @@ -0,0 +1,19 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: true diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-02-01-expected.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-02-01-expected.yaml new file mode 100644 index 00000000..7fd9b38b --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-02-01-expected.yaml @@ -0,0 +1,19 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false From c448c974632432bc384cdcc56a3e85fb1cb0058f Mon Sep 17 00:00:00 2001 From: Vlad Klokun Date: Tue, 10 Jan 2023 19:39:54 +0200 Subject: [PATCH 36/39] tests: test autofixing files with comments between fields --- core/pkg/fixhandler/fixhandler_test.go | 6 ++++++ .../hybrids/tc-03-00-input-comments.yaml | 21 +++++++++++++++++++ .../testdata/hybrids/tc-03-01-expected.yaml | 21 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 core/pkg/fixhandler/testdata/hybrids/tc-03-00-input-comments.yaml create mode 100644 core/pkg/fixhandler/testdata/hybrids/tc-03-01-expected.yaml diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 6bdd293e..5236705c 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -132,6 +132,12 @@ func getTestCases() []indentationTestCase { select(di==0).spec.securityContext.runAsRoot |= false`, "hybrids/tc-02-01-expected.yaml", }, + { + "hybrids/tc-03-00-input-comments.yaml", + `del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`, + "hybrids/tc-03-01-expected.yaml", + }, } return indentationTestCases diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-03-00-input-comments.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-03-00-input-comments.yaml new file mode 100644 index 00000000..ea5c9c51 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-03-00-input-comments.yaml @@ -0,0 +1,21 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + # These are the container comments + containers: + # These are the first containers comments + - name: nginx_container + image: nginx + securityContext: + runAsRoot: true diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-03-01-expected.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-03-01-expected.yaml new file mode 100644 index 00000000..87ef2595 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-03-01-expected.yaml @@ -0,0 +1,21 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + # These are the container comments + containers: + # These are the first containers comments + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false From 5f668037a775c09a59a9c289171a73a61254711a Mon Sep 17 00:00:00 2001 From: Vlad Klokun Date: Tue, 10 Jan 2023 19:41:16 +0200 Subject: [PATCH 37/39] tests: test fixing close to newline-separated keys in hybrid scenarios --- core/pkg/fixhandler/fixhandler_test.go | 6 ++++++ .../tc-04-00-input-separated-keys.yaml | 21 +++++++++++++++++++ .../testdata/hybrids/tc-04-01-expected.yaml | 21 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 core/pkg/fixhandler/testdata/hybrids/tc-04-00-input-separated-keys.yaml create mode 100644 core/pkg/fixhandler/testdata/hybrids/tc-04-01-expected.yaml diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 5236705c..872def5c 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -138,6 +138,12 @@ func getTestCases() []indentationTestCase { select(di==0).spec.securityContext.runAsRoot |= false`, "hybrids/tc-03-01-expected.yaml", }, + { + "hybrids/tc-04-00-input-separated-keys.yaml", + `del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`, + "hybrids/tc-04-01-expected.yaml", + }, } return indentationTestCases diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-04-00-input-separated-keys.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-04-00-input-separated-keys.yaml new file mode 100644 index 00000000..51951d18 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-04-00-input-separated-keys.yaml @@ -0,0 +1,21 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + + image: nginx + + securityContext: + runAsRoot: true diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-04-01-expected.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-04-01-expected.yaml new file mode 100644 index 00000000..485cfa26 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-04-01-expected.yaml @@ -0,0 +1,21 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + + image: nginx + securityContext: + runAsRoot: false + From f1d646ac97e526446a21d485fbcb365820bc535c Mon Sep 17 00:00:00 2001 From: Vlad Klokun Date: Wed, 11 Jan 2023 18:44:06 +0200 Subject: [PATCH 38/39] tests: show diffs when comparing autofixes This change refactors the TestApplyFixKeepsFormatting test to use assert.Equalf so it will display a convenient diff between the expected and actual fixing result. --- core/pkg/fixhandler/fixhandler_test.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 872def5c..4b263ef1 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -9,6 +9,7 @@ import ( metav1 "github.com/kubescape/kubescape/v2/core/meta/datastructures/v1" reporthandlingv2 "github.com/kubescape/opa-utils/reporthandling/v2" "github.com/mikefarah/yq/v4/pkg/yqlib" + "github.com/stretchr/testify/assert" "gopkg.in/op/go-logging.v1" ) @@ -149,7 +150,7 @@ func getTestCases() []indentationTestCase { return indentationTestCases } -func TestApplyFixKeepsIndentation(t *testing.T) { +func TestApplyFixKeepsFormatting(t *testing.T) { testCases := getTestCases() for _, tc := range testCases { @@ -161,24 +162,22 @@ func TestApplyFixKeepsIndentation(t *testing.T) { } input, _ := os.ReadFile(getTestDataPath(tc.inputFile)) - want, _ := os.ReadFile(getTestDataPath(tc.expectedFile)) + wantRaw, _ := os.ReadFile(getTestDataPath(tc.expectedFile)) + want := string(wantRaw) expression := tc.yamlExpression h, _ := NewFixHandlerMock() got, _ := h.ApplyFixToContent(string(input), expression) - if got != string(want) { - t.Errorf( - "Contents of the fixed file don't match the expectation.\n"+ - "FilePath: %s\n\n"+ - "Got:\n<%s>\n\n"+ - "Want:\n<%s>", - tc.inputFile, - got, - want, - ) - } + assert.Equalf( + t, want, got, + "Contents of the fixed file don't match the expectation.\n"+ + "Input file: %s\n\n"+ + "Got: <%s>\n\n"+ + "Want: <%s>", + tc.inputFile, got, want, + ) }, ) From 0e8b2f976dc5fb6dd21e5ebe6611694889df477e Mon Sep 17 00:00:00 2001 From: Vlad Klokun Date: Wed, 11 Jan 2023 18:49:16 +0200 Subject: [PATCH 39/39] tests: extend test cases for autofix inserts This change re-organizes the test cases for inserts performed by the autofixing feature. --- core/pkg/fixhandler/fixhandler_test.go | 29 ++++++++++++++----- ...c-01-00-input-mapping-insert-mapping.yaml} | 0 ...put-mapping-insert-mapping-with-list.yaml} | 0 ...=> tc-03-00-input-list-append-scalar.yaml} | 0 ...l => tc-04-00-input-multiple-inserts.yaml} | 0 ...put-comment-blank-line-single-insert.yaml} | 0 ...-00-input-list-append-scalar-oneline.yaml} | 0 ...=> tc-07-00-input-multiple-documents.yaml} | 0 ...input-mapping-insert-mapping-indented.yaml | 11 +++++++ .../testdata/inserts/tc-08-01-expected.yaml | 15 ++++++++++ ...nput-list-insert-new-mapping-indented.yaml | 11 +++++++ .../testdata/inserts/tc-09-01-expected.yaml | 13 +++++++++ ...c-10-00-input-list-insert-new-mapping.yaml | 11 +++++++ .../testdata/inserts/tc-10-01-expected.yaml | 13 +++++++++ 14 files changed, 96 insertions(+), 7 deletions(-) rename core/pkg/fixhandler/testdata/inserts/{tc-01-00-input.yaml => tc-01-00-input-mapping-insert-mapping.yaml} (100%) rename core/pkg/fixhandler/testdata/inserts/{tc-02-00-input.yaml => tc-02-00-input-mapping-insert-mapping-with-list.yaml} (100%) rename core/pkg/fixhandler/testdata/inserts/{tc-03-00-input.yaml => tc-03-00-input-list-append-scalar.yaml} (100%) rename core/pkg/fixhandler/testdata/inserts/{tc-04-00-input.yaml => tc-04-00-input-multiple-inserts.yaml} (100%) rename core/pkg/fixhandler/testdata/inserts/{tc-05-00-input.yaml => tc-05-00-input-comment-blank-line-single-insert.yaml} (100%) rename core/pkg/fixhandler/testdata/inserts/{tc-06-00-input.yaml => tc-06-00-input-list-append-scalar-oneline.yaml} (100%) rename core/pkg/fixhandler/testdata/inserts/{tc-07-00-input.yaml => tc-07-00-input-multiple-documents.yaml} (100%) create mode 100644 core/pkg/fixhandler/testdata/inserts/tc-08-00-input-mapping-insert-mapping-indented.yaml create mode 100644 core/pkg/fixhandler/testdata/inserts/tc-08-01-expected.yaml create mode 100644 core/pkg/fixhandler/testdata/inserts/tc-09-00-input-list-insert-new-mapping-indented.yaml create mode 100644 core/pkg/fixhandler/testdata/inserts/tc-09-01-expected.yaml create mode 100644 core/pkg/fixhandler/testdata/inserts/tc-10-00-input-list-insert-new-mapping.yaml create mode 100644 core/pkg/fixhandler/testdata/inserts/tc-10-01-expected.yaml diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 4b263ef1..44a26b23 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -40,22 +40,22 @@ func getTestCases() []indentationTestCase { indentationTestCases := []indentationTestCase{ // Insertion Scenarios { - "inserts/tc-01-00-input.yaml", + "inserts/tc-01-00-input-mapping-insert-mapping.yaml", "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", "inserts/tc-01-01-expected.yaml", }, { - "inserts/tc-02-00-input.yaml", + "inserts/tc-02-00-input-mapping-insert-mapping-with-list.yaml", "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]", "inserts/tc-02-01-expected.yaml", }, { - "inserts/tc-03-00-input.yaml", + "inserts/tc-03-00-input-list-append-scalar.yaml", "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", "inserts/tc-03-01-expected.yaml", }, { - "inserts/tc-04-00-input.yaml", + "inserts/tc-04-00-input-multiple-inserts.yaml", `select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation |= false | select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] | @@ -66,23 +66,38 @@ func getTestCases() []indentationTestCase { "inserts/tc-04-01-expected.yaml", }, { - "inserts/tc-05-00-input.yaml", + "inserts/tc-05-00-input-comment-blank-line-single-insert.yaml", "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", "inserts/tc-05-01-expected.yaml", }, { - "inserts/tc-06-00-input.yaml", + "inserts/tc-06-00-input-list-append-scalar-oneline.yaml", "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", "inserts/tc-06-01-expected.yaml", }, { - "inserts/tc-07-00-input.yaml", + "inserts/tc-07-00-input-multiple-documents.yaml", `select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false | select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false`, "inserts/tc-07-01-expected.yaml", }, + { + "inserts/tc-08-00-input-mapping-insert-mapping-indented.yaml", + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]", + "inserts/tc-08-01-expected.yaml", + }, + { + "inserts/tc-09-00-input-list-insert-new-mapping-indented.yaml", + `select(di==0).spec.containers += {"name": "redis", "image": "redis"}`, + "inserts/tc-09-01-expected.yaml", + }, + { + "inserts/tc-10-00-input-list-insert-new-mapping.yaml", + `select(di==0).spec.containers += {"name": "redis", "image": "redis"}`, + "inserts/tc-10-01-expected.yaml", + }, // Removal Scenarios { diff --git a/core/pkg/fixhandler/testdata/inserts/tc-01-00-input.yaml b/core/pkg/fixhandler/testdata/inserts/tc-01-00-input-mapping-insert-mapping.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/inserts/tc-01-00-input.yaml rename to core/pkg/fixhandler/testdata/inserts/tc-01-00-input-mapping-insert-mapping.yaml diff --git a/core/pkg/fixhandler/testdata/inserts/tc-02-00-input.yaml b/core/pkg/fixhandler/testdata/inserts/tc-02-00-input-mapping-insert-mapping-with-list.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/inserts/tc-02-00-input.yaml rename to core/pkg/fixhandler/testdata/inserts/tc-02-00-input-mapping-insert-mapping-with-list.yaml diff --git a/core/pkg/fixhandler/testdata/inserts/tc-03-00-input.yaml b/core/pkg/fixhandler/testdata/inserts/tc-03-00-input-list-append-scalar.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/inserts/tc-03-00-input.yaml rename to core/pkg/fixhandler/testdata/inserts/tc-03-00-input-list-append-scalar.yaml diff --git a/core/pkg/fixhandler/testdata/inserts/tc-04-00-input.yaml b/core/pkg/fixhandler/testdata/inserts/tc-04-00-input-multiple-inserts.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/inserts/tc-04-00-input.yaml rename to core/pkg/fixhandler/testdata/inserts/tc-04-00-input-multiple-inserts.yaml diff --git a/core/pkg/fixhandler/testdata/inserts/tc-05-00-input.yaml b/core/pkg/fixhandler/testdata/inserts/tc-05-00-input-comment-blank-line-single-insert.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/inserts/tc-05-00-input.yaml rename to core/pkg/fixhandler/testdata/inserts/tc-05-00-input-comment-blank-line-single-insert.yaml diff --git a/core/pkg/fixhandler/testdata/inserts/tc-06-00-input.yaml b/core/pkg/fixhandler/testdata/inserts/tc-06-00-input-list-append-scalar-oneline.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/inserts/tc-06-00-input.yaml rename to core/pkg/fixhandler/testdata/inserts/tc-06-00-input-list-append-scalar-oneline.yaml diff --git a/core/pkg/fixhandler/testdata/inserts/tc-07-00-input.yaml b/core/pkg/fixhandler/testdata/inserts/tc-07-00-input-multiple-documents.yaml similarity index 100% rename from core/pkg/fixhandler/testdata/inserts/tc-07-00-input.yaml rename to core/pkg/fixhandler/testdata/inserts/tc-07-00-input-multiple-documents.yaml diff --git a/core/pkg/fixhandler/testdata/inserts/tc-08-00-input-mapping-insert-mapping-indented.yaml b/core/pkg/fixhandler/testdata/inserts/tc-08-00-input-mapping-insert-mapping-indented.yaml new file mode 100644 index 00000000..ac27db64 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-08-00-input-mapping-insert-mapping-indented.yaml @@ -0,0 +1,11 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] + +apiVersion: v1 +kind: Pod +metadata: + name: indented-parent-list-insert-list-value +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/inserts/tc-08-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-08-01-expected.yaml new file mode 100644 index 00000000..46a3acbe --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-08-01-expected.yaml @@ -0,0 +1,15 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] + +apiVersion: v1 +kind: Pod +metadata: + name: indented-parent-list-insert-list-value +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + capabilities: + drop: + - NET_RAW diff --git a/core/pkg/fixhandler/testdata/inserts/tc-09-00-input-list-insert-new-mapping-indented.yaml b/core/pkg/fixhandler/testdata/inserts/tc-09-00-input-list-insert-new-mapping-indented.yaml new file mode 100644 index 00000000..416c0c1a --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-09-00-input-list-insert-new-mapping-indented.yaml @@ -0,0 +1,11 @@ +# Fix to Apply: +# select(di==0).spec.containers += {"name": "redis", "image": "redis"} + +apiVersion: v1 +kind: Pod +metadata: + name: indented-parent-list-insert-list-value +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/inserts/tc-09-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-09-01-expected.yaml new file mode 100644 index 00000000..61ce65f7 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-09-01-expected.yaml @@ -0,0 +1,13 @@ +# Fix to Apply: +# select(di==0).spec.containers += {"name": "redis", "image": "redis"} + +apiVersion: v1 +kind: Pod +metadata: + name: indented-parent-list-insert-list-value +spec: + containers: + - name: nginx_container + image: nginx + - name: redis + image: redis diff --git a/core/pkg/fixhandler/testdata/inserts/tc-10-00-input-list-insert-new-mapping.yaml b/core/pkg/fixhandler/testdata/inserts/tc-10-00-input-list-insert-new-mapping.yaml new file mode 100644 index 00000000..827cad1d --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-10-00-input-list-insert-new-mapping.yaml @@ -0,0 +1,11 @@ +# Fix to Apply: +# select(di==0).spec.containers += {"name": "redis", "image": "redis"} + +apiVersion: v1 +kind: Pod +metadata: + name: indented-list-insert-new-object +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/inserts/tc-10-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-10-01-expected.yaml new file mode 100644 index 00000000..a46f6b75 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-10-01-expected.yaml @@ -0,0 +1,13 @@ +# Fix to Apply: +# select(di==0).spec.containers += {"name": "redis", "image": "redis"} + +apiVersion: v1 +kind: Pod +metadata: + name: indented-list-insert-new-object +spec: + containers: + - name: nginx_container + image: nginx + - name: redis + image: redis