Initial Implementation

This commit is contained in:
suhasgumma
2022-12-08 22:56:53 +05:30
parent f3665866af
commit 4b898b0075
5 changed files with 463 additions and 207 deletions
+32 -3
View File
@@ -5,6 +5,7 @@ import (
metav1 "github.com/kubescape/kubescape/v2/core/meta/datastructures/v1"
"github.com/kubescape/opa-utils/reporthandling"
reporthandlingv2 "github.com/kubescape/opa-utils/reporthandling/v2"
"gopkg.in/yaml.v3"
)
// FixHandler is a struct that holds the information of the report to be fixed
@@ -21,8 +22,36 @@ type ResourceFixInfo struct {
FilePath string
}
// LineAndContentToAdd holds the information about where to insert the new changes in the existing yaml file
type LineAndContentToAdd struct {
Line int
// NodeInfo holds extra information about the node
type NodeInfo struct {
node *yaml.Node
parent *yaml.Node
// position of the node among siblings
index int
}
// FixInfoMetadata holds the arguments "getFixInfo" function needs to pass to the
// functions it uses
type FixInfoMetadata struct {
originalList *[]NodeInfo
fixedList *[]NodeInfo
originalListTracker int
fixedListTracker int
contentToAdd *[]ContentToAdd
contentToRemove *[]ContentToRemove
}
// ContentToAdd holds the information about where to insert the new changes in the existing yaml file
type ContentToAdd struct {
// Line where the fix should be applied to
Line int
// Content is a string representation of the YAML node that describes a suggested fix
Content string
}
// ContentToRemove holds the line numbers to remove from the existing yaml file
type ContentToRemove struct {
startLine int
endLine int
}
+6 -2
View File
@@ -216,9 +216,13 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str
}
func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError error) {
originalYamlNode := getDecodedYaml(filePath)
fixedYamlNode := getFixedYamlNode(filePath, yamlExpression)
lineAndContentsToAdd := getLineAndContentToAdd(&fixedYamlNode)
err := addFixesToFile(filePath, *lineAndContentsToAdd)
originalList := getDFSOrder(originalYamlNode)
fixedList := getDFSOrder(fixedYamlNode)
contentToAdd, linesToRemove := getFixInfo(originalList, fixedList)
err := applyFixesToFile(filePath, contentToAdd, linesToRemove)
return err
}
+146 -188
View File
@@ -4,205 +4,187 @@ import (
"bufio"
"bytes"
"container/list"
"errors"
"fmt"
"io"
"log"
"io/ioutil"
"os"
"strings"
logger "github.com/kubescape/go-logger"
"github.com/mikefarah/yq/v4/pkg/yqlib"
"gopkg.in/yaml.v3"
)
func getFixedYamlNode(filePath, yamlExpression string) yaml.Node {
func getDecodedYaml(filepath string) *yaml.Node {
file, err := ioutil.ReadFile(filepath)
if err != nil {
logger.L().Fatal("Cannot read file")
}
fileReader := bytes.NewReader(file)
dec := yaml.NewDecoder(fileReader)
var node yaml.Node
err = dec.Decode(&node)
if err != nil {
logger.L().Fatal("Cannot Decode Yaml")
}
return &node
}
func getFixedYamlNode(filePath, yamlExpression string) *yaml.Node {
preferences := yqlib.ConfiguredYamlPreferences
preferences.EvaluateTogether = true
decoder := yqlib.NewYamlDecoder(preferences)
var allDocuments = list.New()
reader, err := readStream(filePath)
reader, err := getNewReader(filePath)
if err != nil {
return yaml.Node{}
return &yaml.Node{}
}
fileDocuments, err := readDocuments(reader, filePath, 0, decoder)
if err != nil {
return yaml.Node{}
return &yaml.Node{}
}
allDocuments.PushBackList(fileDocuments)
if allDocuments.Len() == 0 {
candidateNode := &yqlib.CandidateNode{
Document: 0,
Filename: "",
Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
FileIndex: 0,
LeadingContent: "",
}
allDocuments.PushBack(candidateNode)
}
allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
matches, _ := allAtOnceEvaluator.EvaluateCandidateNodes(yamlExpression, allDocuments)
matches, err := allAtOnceEvaluator.EvaluateCandidateNodes(yamlExpression, allDocuments)
return *matches.Front().Value.(*yqlib.CandidateNode).Node
}
func readStream(filename string) (io.Reader, error) {
var reader *bufio.Reader
if filename == "-" {
reader = bufio.NewReader(os.Stdin)
} else {
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
// and ensuring that it's not possible to give a path to a file outside thar directory.
file, err := os.Open(filename) // #nosec
if err != nil {
return nil, err
}
reader = bufio.NewReader(file)
}
return reader, nil
}
func readDocuments(reader io.Reader, filename string, fileIndex int, decoder yqlib.Decoder) (*list.List, error) {
err := decoder.Init(reader)
if err != nil {
return nil, err
logger.L().Fatal(fmt.Sprintf("Error fixing YAML, %v", err.Error()))
}
inputList := list.New()
var currentIndex uint
for {
candidateNode, errorReading := decoder.Decode()
return matches.Front().Value.(*yqlib.CandidateNode).Node
}
if errors.Is(errorReading, io.EOF) {
switch reader := reader.(type) {
case *os.File:
safelyCloseFile(reader)
}
return inputList, nil
} else if errorReading != nil {
return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading)
}
candidateNode.Document = currentIndex
candidateNode.Filename = filename
candidateNode.FileIndex = fileIndex
candidateNode.EvaluateTogether = true
func getDFSOrder(node *yaml.Node) *[]NodeInfo {
dfsOrder := make([]NodeInfo, 0)
getDFSOrderHelper(node, nil, &dfsOrder, 0)
return &dfsOrder
}
inputList.PushBack(candidateNode)
func matchNodes(nodeOne, nodeTwo *yaml.Node) int {
currentIndex = currentIndex + 1
isNewNode := nodeTwo.Line == 0 && nodeTwo.Column == 0
sameLines := nodeOne.Line == nodeTwo.Line
sameColumns := nodeOne.Column == nodeTwo.Column
sameKinds := nodeOne.Kind == nodeTwo.Kind
sameValues := nodeOne.Value == nodeTwo.Value
isSameNode := sameKinds && sameValues && sameLines && sameColumns
switch {
case isSameNode:
return int(sameNodes)
case isNewNode:
return int(insertedNode)
case sameLines && sameColumns:
return int(replacedNode)
default:
return int(removedNode)
}
}
func safelyCloseFile(file *os.File) {
err := file.Close()
if err != nil {
fmt.Println("Error Closing File")
func getFixInfo(originalList, fixedList *[]NodeInfo) (*[]ContentToAdd, *[]ContentToRemove) {
contentToAdd := make([]ContentToAdd, 0)
linesToRemove := make([]ContentToRemove, 0)
originalListTracker, fixedListTracker := 0, 0
fixInfoMetadata := &FixInfoMetadata{
originalList: originalList,
fixedList: fixedList,
originalListTracker: originalListTracker,
fixedListTracker: fixedListTracker,
contentToAdd: &contentToAdd,
contentToRemove: &linesToRemove,
}
for originalListTracker < len(*originalList) && fixedListTracker < len(*fixedList) {
matchNodeResult := matchNodes((*originalList)[originalListTracker].node, (*fixedList)[fixedListTracker].node)
fixInfoMetadata.originalListTracker = originalListTracker
fixInfoMetadata.fixedListTracker = fixedListTracker
switch matchNodeResult {
case int(sameNodes):
originalListTracker += 1
fixedListTracker += 1
case int(removedNode):
originalListTracker = addLinesToRemove(fixInfoMetadata)
case int(insertedNode):
fixedListTracker = addLinesToInsert(fixInfoMetadata)
case int(replacedNode):
originalListTracker, fixedListTracker = updateLinesToReplace(fixInfoMetadata)
}
}
for originalListTracker < len(*originalList) {
fixInfoMetadata.originalListTracker = originalListTracker
fixInfoMetadata.fixedListTracker = len(*fixedList) - 1
originalListTracker = addLinesToRemove(fixInfoMetadata)
}
for fixedListTracker < len(*fixedList) {
fixInfoMetadata.originalListTracker = len(*originalList) - 1
fixInfoMetadata.fixedListTracker = fixedListTracker
fixedListTracker = addLinesToInsert(fixInfoMetadata)
}
return &contentToAdd, &linesToRemove
}
func getLineAndContentToAdd(node *yaml.Node) *[]LineAndContentToAdd {
contentToAdd := make([]LineAndContentToAdd, 0)
getLineAndContentToAddHelper(0, node, &contentToAdd)
return &contentToAdd
// Adds the lines to remove and returns the updated originalListTracker
func addLinesToRemove(fixInfoMetadata *FixInfoMetadata) int {
currentDFSNode := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker]
newTracker := updateTracker(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker)
*fixInfoMetadata.contentToRemove = append(*fixInfoMetadata.contentToRemove, ContentToRemove{
startLine: currentDFSNode.node.Line,
endLine: getNodeLine(fixInfoMetadata.originalList, newTracker) - 1,
})
return newTracker
}
func getLineAndContentToAddHelper(nodeIdx int, parentNode *yaml.Node, contentToAdd *[]LineAndContentToAdd) {
node := parentNode.Content[nodeIdx]
var content string
var err error
if node.Line == 0 && node.Column == 0 {
if parentNode.Kind == yaml.MappingNode {
if nodeIdx%2 != 0 {
return
}
}
content, err = enocodeIntoYaml(parentNode, nodeIdx)
// Adds the lines to insert and returns the updated fixedListTracker
func addLinesToInsert(fixInfoMetadata *FixInfoMetadata) int {
currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker]
lineToInsert := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker].node.Line - 1
contentToInsert := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker)
if err != nil {
fmt.Println("Cannot Encode into YAML")
}
newTracker := updateTracker(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker)
indentationSpacesBeforeContent := parentNode.Column - 1
*fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, ContentToAdd{
Line: lineToInsert,
Content: contentToInsert,
})
content = addIndentationToContent(content, indentationSpacesBeforeContent)
// Getting the line to add content after. Add directly after the left Sibling.
var lineToAddAfter int
for idx := nodeIdx - 1; idx >= 0; idx-- {
if parentNode.Content[idx].Line != 0 {
lineToAddAfter = getEndingLine(idx, parentNode)
}
}
lineAndContentToAdd := LineAndContentToAdd{
Line: lineToAddAfter,
Content: content,
}
*contentToAdd = append(*contentToAdd, lineAndContentToAdd)
}
for index, _ := range node.Content {
getLineAndContentToAddHelper(index, node, contentToAdd)
}
return newTracker
}
func enocodeIntoYaml(parentNode *yaml.Node, nodeIdx int) (string, error) {
if parentNode.Kind == yaml.MappingNode {
content := make([]*yaml.Node, 0)
content = append(content, parentNode.Content[nodeIdx], parentNode.Content[nodeIdx+1])
parentForContent := yaml.Node{
Kind: yaml.MappingNode,
Content: content,
}
buf := new(bytes.Buffer)
encoder := yaml.NewEncoder(buf)
errorEncoding := encoder.Encode(parentForContent)
if errorEncoding != nil {
return "", fmt.Errorf("Error debugging node, %v", errorEncoding.Error())
}
errorClosingEncoder := encoder.Close()
if errorClosingEncoder != nil {
return "", fmt.Errorf("Error closing encoder: ", errorClosingEncoder.Error())
}
return fmt.Sprintf(`%v`, buf.String()), nil
// Adds the lines to remove and insert and updates the fixedListTracker and originalListTracker
func updateLinesToReplace(fixInfoMetadata *FixInfoMetadata) (int, int) {
currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker]
if isValueNodeinMapping(&currentDFSNode) {
fixInfoMetadata.originalListTracker -= 1
fixInfoMetadata.fixedListTracker -= 1
}
return "", nil
updatedOriginalTracker := addLinesToRemove(fixInfoMetadata)
updatedFixedTracker := addLinesToInsert(fixInfoMetadata)
return updatedOriginalTracker, updatedFixedTracker
}
func addIndentationToContent(content string, indentationSpacesBeforeContent int) string {
indentedContent := ""
indentSpaces := strings.Repeat(" ", indentationSpacesBeforeContent)
scanner := bufio.NewScanner(strings.NewReader(content))
for scanner.Scan() {
line := scanner.Text()
indentedContent += (indentSpaces + line + "\n")
}
return indentedContent
}
func getEndingLine(nodeIdx int, parentNode *yaml.Node) int {
node := parentNode.Content[nodeIdx]
if node.Kind == yaml.ScalarNode {
return node.Line
}
contentLen := len(node.Content)
for idx := contentLen - 1; idx >= 0; idx-- {
if node.Content[idx].Line != 0 {
return getEndingLine(idx, node)
}
}
return 0
}
func addFixesToFile(filePath string, lineAndContentsToAdd []LineAndContentToAdd) (cmdError error) {
func applyFixesToFile(filePath string, lineAndContentsToAdd *[]ContentToAdd, linesToRemove *[]ContentToRemove) (cmdError error) {
linesSlice, err := getLinesSlice(filePath)
if err != nil {
@@ -225,11 +207,16 @@ func addFixesToFile(filePath string, lineAndContentsToAdd []LineAndContentToAdd)
return nil
}()
removeLines(linesToRemove, &linesSlice)
writer := bufio.NewWriter(file)
lineIdx, lineToAddIdx := 0, 0
for lineToAddIdx < len(lineAndContentsToAdd) {
for lineIdx <= lineAndContentsToAdd[lineToAddIdx].Line {
for lineToAddIdx < len(*lineAndContentsToAdd) {
for lineIdx <= (*lineAndContentsToAdd)[lineToAddIdx].Line {
if linesSlice[lineIdx] == "*" {
continue
}
_, err := writer.WriteString(linesSlice[lineIdx] + "\n")
if err != nil {
return err
@@ -237,11 +224,14 @@ func addFixesToFile(filePath string, lineAndContentsToAdd []LineAndContentToAdd)
lineIdx += 1
}
writeContentToAdd(writer, lineAndContentsToAdd[lineToAddIdx].Content)
writeContentToAdd(writer, (*lineAndContentsToAdd)[lineToAddIdx].Content)
lineToAddIdx += 1
}
for lineIdx < len(linesSlice) {
if linesSlice[lineIdx] == "*" {
continue
}
_, err := writer.WriteString(linesSlice[lineIdx] + "\n")
if err != nil {
return err
@@ -252,35 +242,3 @@ func addFixesToFile(filePath string, lineAndContentsToAdd []LineAndContentToAdd)
writer.Flush()
return nil
}
// Get the lines of existing yaml in a slice
func getLinesSlice(filePath string) ([]string, error) {
lineSlice := make([]string, 0)
file, err := os.Open(filePath)
if err != nil {
log.Fatal(err)
return nil, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lineSlice = append(lineSlice, scanner.Text())
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
return nil, err
}
return lineSlice, err
}
func writeContentToAdd(writer *bufio.Writer, contentToAdd string) {
scanner := bufio.NewScanner(strings.NewReader(contentToAdd))
for scanner.Scan() {
line := scanner.Text()
writer.WriteString(line + "\n")
}
}
+276
View File
@@ -0,0 +1,276 @@
package fixhandler
import (
"bufio"
"bytes"
"container/list"
"errors"
"fmt"
"io"
"log"
"math"
"os"
"strings"
logger "github.com/kubescape/go-logger"
"github.com/mikefarah/yq/v4/pkg/yqlib"
"gopkg.in/yaml.v3"
)
const (
sameNodes = iota
insertedNode
removedNode
replacedNode
)
func getNewReader(filename string) (io.Reader, error) {
var reader *bufio.Reader
if filename == "-" {
reader = bufio.NewReader(os.Stdin)
} else {
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
// and ensuring that it's not possible to give a path to a file outside thar directory.
file, err := os.Open(filename) // #nosec
if err != nil {
return nil, err
}
reader = bufio.NewReader(file)
}
return reader, nil
}
func readDocuments(reader io.Reader, filename string, fileIndex int, decoder yqlib.Decoder) (*list.List, error) {
err := decoder.Init(reader)
if err != nil {
return nil, err
}
inputList := list.New()
var currentIndex uint
for {
candidateNode, errorReading := decoder.Decode()
if errors.Is(errorReading, io.EOF) {
switch reader := reader.(type) {
case *os.File:
safelyCloseFile(reader)
}
return inputList, nil
} else if errorReading != nil {
return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading)
}
candidateNode.Document = currentIndex
candidateNode.Filename = filename
candidateNode.FileIndex = fileIndex
candidateNode.EvaluateTogether = true
inputList.PushBack(candidateNode)
currentIndex = currentIndex + 1
}
}
func safelyCloseFile(file *os.File) {
err := file.Close()
if err != nil {
logger.L().Error("Error Closing File")
}
}
func getDFSOrderHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]NodeInfo, index int) {
dfsNode := NodeInfo{
node: node,
parent: parent,
index: index,
}
fmt.Println(dfsNode.node)
fmt.Println(dfsNode.parent)
fmt.Println(dfsNode.index)
*dfsOrder = append(*dfsOrder, dfsNode)
for idx, child := range node.Content {
getDFSOrderHelper(child, node, dfsOrder, idx)
}
}
// Skips the current node including it's children in DFS order and returns the new tracker.
func skipCurrentNode(node *yaml.Node, currentTracker int) int {
updatedTracker := currentTracker + getChildrenCount(node)
return updatedTracker
}
func getChildrenCount(node *yaml.Node) int {
totalChildren := 1
for _, child := range node.Content {
totalChildren += getChildrenCount(child)
}
return totalChildren
}
// Moves the tracker to the parent of given node
func traceBackToParent(dfsOrder *[]NodeInfo, currentTracker int) int {
parentNode := (*dfsOrder)[currentTracker].parent
parentIdx := currentTracker - 1
for parentIdx >= 0 {
if (*dfsOrder)[parentIdx].node == parentNode {
return parentIdx
}
parentIdx -= 1
}
return 0
}
// Checks if the node is value node in "key-value" pairs of mapping node
func isValueNodeinMapping(dfsNode *NodeInfo) bool {
if dfsNode.parent.Kind == yaml.MappingNode && dfsNode.index%2 != 0 {
return true
}
return false
}
func updateTracker(dfsOrder *[]NodeInfo, tracker int) int {
currentDFSNode := (*dfsOrder)[tracker]
var newTracker int
if currentDFSNode.parent.Kind == yaml.MappingNode {
valueNode := (*dfsOrder)[tracker+1]
newTracker = skipCurrentNode(valueNode.node, tracker+1)
} else {
newTracker = skipCurrentNode(currentDFSNode.node, tracker)
}
return newTracker
}
func getNodeLine(dfsOrder *[]NodeInfo, tracker int) int {
if tracker < len(*dfsOrder) {
return (*dfsOrder)[tracker].node.Line
} else {
return int(math.Inf(1))
}
}
func isOneLineSequenceNode(node *yaml.Node) bool {
if node.Kind != yaml.SequenceNode {
return false
}
nodeLine := node.Line
for _, child := range node.Content {
if child.Line != nodeLine {
return false
}
}
return true
}
func enocodeIntoYaml(parentNode *yaml.Node, dfsOrder *[]NodeInfo, tracker int) (string, error) {
content := make([]*yaml.Node, 0)
currentNode := (*dfsOrder)[tracker].node
content = append(content, currentNode)
if parentNode.Kind == yaml.MappingNode {
valueNode := (*dfsOrder)[tracker+1].node
content = append(content, valueNode)
}
parentForContent := yaml.Node{
Kind: parentNode.Kind,
Content: content,
}
buf := new(bytes.Buffer)
encoder := yaml.NewEncoder(buf)
errorEncoding := encoder.Encode(parentForContent)
if errorEncoding != nil {
return "", fmt.Errorf("Error debugging node, %v", errorEncoding.Error())
}
errorClosingEncoder := encoder.Close()
if errorClosingEncoder != nil {
return "", fmt.Errorf("Error closing encoder: %v", errorClosingEncoder.Error())
}
return fmt.Sprintf(`%v`, buf.String()), nil
}
func getContent(parentNode *yaml.Node, dfsOrder *[]NodeInfo, tracker int) string {
content, err := enocodeIntoYaml(parentNode, dfsOrder, tracker)
if err != nil {
logger.L().Fatal("Cannot Encode into YAML")
}
indentationSpaces := parentNode.Column - 1
content = indentContent(content, indentationSpaces)
return content
}
func indentContent(content string, indentationSpaces int) string {
indentedContent := ""
indentSpaces := strings.Repeat(" ", indentationSpaces)
scanner := bufio.NewScanner(strings.NewReader(content))
for scanner.Scan() {
line := scanner.Text()
indentedContent += (indentSpaces + line + "\n")
}
return indentedContent
}
func removeLines(linesToRemove *[]ContentToRemove, linesSlice *[]string) {
for _, lineToRemove := range *linesToRemove {
startLine := lineToRemove.startLine
endLine := int(math.Min(float64(lineToRemove.endLine), float64(len(*linesSlice)-1)))
for line := startLine; line <= endLine; line++ {
lineContent := strings.ReplaceAll((*linesSlice)[line], " ", "")
if isEmptyLineOrComment(lineContent) {
break
}
(*linesSlice)[line] = "*"
}
}
}
// Checks if the line is empty or a comment
func isEmptyLineOrComment(lineContent string) bool {
if lineContent == "" {
return true
} else if lineContent[0:1] == "#" {
return true
}
return false
}
// Get the lines of existing yaml in a slice
func getLinesSlice(filePath string) ([]string, error) {
lineSlice := make([]string, 0)
file, err := os.Open(filePath)
if err != nil {
log.Fatal(err)
return nil, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lineSlice = append(lineSlice, scanner.Text())
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
return nil, err
}
return lineSlice, err
}
func writeContentToAdd(writer *bufio.Writer, contentToAdd string) {
scanner := bufio.NewScanner(strings.NewReader(contentToAdd))
for scanner.Scan() {
line := scanner.Text()
writer.WriteString(line + "\n")
}
}
+3 -14
View File
@@ -1,17 +1,3 @@
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -58,6 +44,9 @@ spec:
periodSeconds: 15
exec:
command: ["/bin/grpc_health_probe", "-addr=:9555"]
securityContext:
allowPrivilegeEscalation: false
---
apiVersion: v1
kind: Service