diff --git a/core/cautils/scaninfo.go b/core/cautils/scaninfo.go index 53ca9107..5f32bf04 100644 --- a/core/cautils/scaninfo.go +++ b/core/cautils/scaninfo.go @@ -419,7 +419,7 @@ func metadataGitLocal(input string) (*reporthandlingv2.RepoContextMetadata, erro Date: commit.Committer.Date, CommitterName: commit.Committer.Name, } - context.LocalRootPath = getAbsPath(input) + context.LocalRootPath, _ = gitParser.GetRootDir() return context, nil } diff --git a/core/pkg/fixhandler/datastructures.go b/core/pkg/fixhandler/datastructures.go index 6b2601ae..c844009c 100644 --- a/core/pkg/fixhandler/datastructures.go +++ b/core/pkg/fixhandler/datastructures.go @@ -5,6 +5,7 @@ import ( metav1 "github.com/kubescape/kubescape/v2/core/meta/datastructures/v1" "github.com/kubescape/opa-utils/reporthandling" reporthandlingv2 "github.com/kubescape/opa-utils/reporthandling/v2" + "gopkg.in/yaml.v3" ) // FixHandler is a struct that holds the information of the report to be fixed @@ -19,4 +20,44 @@ type ResourceFixInfo struct { YamlExpressions map[string]*armotypes.FixPath Resource *reporthandling.Resource FilePath string + DocumentIndex int +} + +// NodeInfo holds extra information about the node +type nodeInfo struct { + node *yaml.Node + parent *yaml.Node + + // position of the node among siblings + index int +} + +// FixInfoMetadata holds the arguments "getFixInfo" function needs to pass to the +// functions it uses +type fixInfoMetadata struct { + originalList *[]nodeInfo + fixedList *[]nodeInfo + originalListTracker int + fixedListTracker int + contentToAdd *[]contentToAdd + linesToRemove *[]linesToRemove +} + +// ContentToAdd holds the information about where to insert the new changes in the existing yaml file +type contentToAdd struct { + // Line where the fix should be applied to + line int + // Content is a string representation of the YAML node that describes a suggested fix + content string +} + +// LinesToRemove holds the line numbers to remove from the existing yaml file +type linesToRemove struct { + startLine int + endLine int +} + +type fileFixInfo struct { + contentsToAdd *[]contentToAdd + linesToRemove *[]linesToRemove } diff --git a/core/pkg/fixhandler/fixhandler.go b/core/pkg/fixhandler/fixhandler.go index 26005b20..454bc201 100644 --- a/core/pkg/fixhandler/fixhandler.go +++ b/core/pkg/fixhandler/fixhandler.go @@ -145,6 +145,7 @@ func (h *FixHandler) PrepareResourcesToFix() []ResourceFixInfo { FilePath: absolutePath, Resource: resourceObj, YamlExpressions: make(map[string]*armotypes.FixPath, 0), + DocumentIndex: documentIndex, } for i := range result.AssociatedControls { @@ -185,19 +186,34 @@ func (h *FixHandler) PrintExpectedChanges(resourcesToFix []ResourceFixInfo) { func (h *FixHandler) ApplyChanges(resourcesToFix []ResourceFixInfo) (int, []error) { updatedFiles := make(map[string]bool) errors := make([]error, 0) - for _, resourceToFix := range resourcesToFix { - singleExpression := reduceYamlExpressions(&resourceToFix) - if err := h.applyFixToFile(resourceToFix.FilePath, singleExpression); err != nil { - errors = append(errors, - fmt.Errorf("failed to fix resource [Name: '%s', Kind: '%s'] in '%s': %w ", - resourceToFix.Resource.GetName(), - resourceToFix.Resource.GetKind(), - resourceToFix.FilePath, - err)) + + fileYamlExpressions := h.getFileYamlExpressions(resourcesToFix) + + for filepath, yamlExpression := range fileYamlExpressions { + fileAsString, err := getFileString(filepath) + + if err != nil { + errors = append(errors, err) + continue + } + + fixedYamlString, err := h.ApplyFixToContent(fileAsString, yamlExpression) + + if err != nil { + errors = append(errors, fmt.Errorf("Failed to fix file %s: %w ", filepath, err)) + continue } else { - updatedFiles[resourceToFix.FilePath] = true + updatedFiles[filepath] = true + } + + err = writeFixesToFile(filepath, fixedYamlString) + + if err != nil { + logger.L().Error(fmt.Sprintf("Failed to write fixes to file %s, %v", filepath, err.Error())) + errors = append(errors, err) } } + return len(updatedFiles), errors } @@ -215,34 +231,45 @@ func (h *FixHandler) getFilePathAndIndex(filePathWithIndex string) (filePath str } } -func (h *FixHandler) applyFixToFile(filePath, yamlExpression string) (cmdError error) { - var completedSuccessfully bool - writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(filePath) - out, err := writeInPlaceHandler.CreateTempFile() +func (h *FixHandler) ApplyFixToContent(yamlAsString, yamlExpression string) (fixedString string, err error) { + yamlLines := strings.Split(yamlAsString, "\n") + + originalRootNodes, err := decodeDocumentRoots(yamlAsString) + if err != nil { - return fmt.Errorf("unable to create a tmp file for in-place YAML update: %s", err) + return "", err } - defer func() { - if cmdError == nil { - cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) + fixedRootNodes, err := getFixedNodes(yamlAsString, yamlExpression) + + if err != nil { + return "", err + } + + fileFixInfo := getFixInfo(originalRootNodes, fixedRootNodes) + + fixedYamlLines := getFixedYamlLines(yamlLines, fileFixInfo) + + fixedString = getStringFromSlice(fixedYamlLines) + + return fixedString, nil +} + +func (h *FixHandler) getFileYamlExpressions(resourcesToFix []ResourceFixInfo) map[string]string { + fileYamlExpressions := make(map[string]string, 0) + for _, resourceToFix := range resourcesToFix { + singleExpression := reduceYamlExpressions(&resourceToFix) + resourceFilePath := resourceToFix.FilePath + + if _, pathExistsInMap := fileYamlExpressions[resourceFilePath]; !pathExistsInMap { + fileYamlExpressions[resourceFilePath] = singleExpression + } else { + fileYamlExpressions[resourceFilePath] = joinStrings(fileYamlExpressions[resourceFilePath], " | ", singleExpression) } - }() - encoder := yqlib.NewYamlEncoder(2, false, yqlib.ConfiguredYamlPreferences) + } - printer := yqlib.NewPrinter(encoder, yqlib.NewSinglePrinterWriter(out)) - allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator() - - preferences := yqlib.ConfiguredYamlPreferences - preferences.EvaluateTogether = true - decoder := yqlib.NewYamlDecoder(preferences) - - err = allAtOnceEvaluator.EvaluateFiles(yamlExpression, []string{filePath}, printer, decoder) - - completedSuccessfully = err == nil - - return err + return fileYamlExpressions } func (rfi *ResourceFixInfo) addYamlExpressionsFromResourceAssociatedControl(documentIndex int, ac *resourcesresults.ResourceAssociatedControl, skipUserValues bool) { @@ -293,3 +320,27 @@ func fixPathToValidYamlExpression(fixPath, value string, documentIndexInYaml int // select document index and add a dot for the root node return fmt.Sprintf("select(di==%d).%s |= %s", documentIndexInYaml, fixPath, value) } + +func joinStrings(inputStrings ...string) string { + return strings.Join(inputStrings, "") +} + +func getFileString(filepath string) (string, error) { + bytes, err := ioutil.ReadFile(filepath) + + if err != nil { + return "", fmt.Errorf("Error reading file %s", filepath) + } + + return string(bytes), nil +} + +func writeFixesToFile(filepath, content string) error { + err := ioutil.WriteFile(filepath, []byte(content), 0644) + + if err != nil { + return fmt.Errorf("Error writing fixes to file: %w", err) + } + + return nil +} diff --git a/core/pkg/fixhandler/fixhandler_test.go b/core/pkg/fixhandler/fixhandler_test.go index 9073361c..44a26b23 100644 --- a/core/pkg/fixhandler/fixhandler_test.go +++ b/core/pkg/fixhandler/fixhandler_test.go @@ -1,20 +1,24 @@ package fixhandler import ( - "io/ioutil" "os" "path/filepath" "testing" - "github.com/stretchr/testify/assert" - logger "github.com/kubescape/go-logger" metav1 "github.com/kubescape/kubescape/v2/core/meta/datastructures/v1" reporthandlingv2 "github.com/kubescape/opa-utils/reporthandling/v2" "github.com/mikefarah/yq/v4/pkg/yqlib" + "github.com/stretchr/testify/assert" "gopkg.in/op/go-logging.v1" ) +type indentationTestCase struct { + inputFile string + yamlExpression string + expectedFile string +} + func NewFixHandlerMock() (*FixHandler, error) { backendLoggerLeveled := logging.AddModuleLevel(logging.NewLogBackend(logger.L().GetWriter(), "", 0)) backendLoggerLeveled.SetLevel(logging.ERROR, "") @@ -27,45 +31,172 @@ func NewFixHandlerMock() (*FixHandler, error) { }, nil } -func onlineBoutiquePath() string { - o, _ := os.Getwd() - return filepath.Join(filepath.Dir(o), "..", "..", "examples", "online-boutique") +func getTestdataPath() string { + currentDir, _ := os.Getwd() + return filepath.Join(currentDir, "testdata") } -func TestFixHandler_applyFixToFile(t *testing.T) { - originalFilePath := filepath.Join(onlineBoutiquePath(), "adservice.yaml") - // create temp file - tempFile, err := ioutil.TempFile("", "adservice.yaml") - if err != nil { - panic(err) - } - defer os.Remove(tempFile.Name()) +func getTestCases() []indentationTestCase { + indentationTestCases := []indentationTestCase{ + // Insertion Scenarios + { + "inserts/tc-01-00-input-mapping-insert-mapping.yaml", + "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + "inserts/tc-01-01-expected.yaml", + }, + { + "inserts/tc-02-00-input-mapping-insert-mapping-with-list.yaml", + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]", + "inserts/tc-02-01-expected.yaml", + }, + { + "inserts/tc-03-00-input-list-append-scalar.yaml", + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", + "inserts/tc-03-01-expected.yaml", + }, + { + "inserts/tc-04-00-input-multiple-inserts.yaml", - // read original file - b, err := ioutil.ReadFile(originalFilePath) - if err != nil { - panic(err) - } - assert.NotContains(t, string(b), "readOnlyRootFilesystem: true") + `select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation |= false | + select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] | + select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type |= "RuntimeDefault" | + select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false | + select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true`, - // write original file contents to temp file - err = ioutil.WriteFile(tempFile.Name(), b, 0644) - if err != nil { - panic(err) + "inserts/tc-04-01-expected.yaml", + }, + { + "inserts/tc-05-00-input-comment-blank-line-single-insert.yaml", + "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false", + "inserts/tc-05-01-expected.yaml", + }, + { + "inserts/tc-06-00-input-list-append-scalar-oneline.yaml", + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"SYS_ADM\"]", + "inserts/tc-06-01-expected.yaml", + }, + { + "inserts/tc-07-00-input-multiple-documents.yaml", + + `select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false | + select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false`, + + "inserts/tc-07-01-expected.yaml", + }, + { + "inserts/tc-08-00-input-mapping-insert-mapping-indented.yaml", + "select(di==0).spec.containers[0].securityContext.capabilities.drop += [\"NET_RAW\"]", + "inserts/tc-08-01-expected.yaml", + }, + { + "inserts/tc-09-00-input-list-insert-new-mapping-indented.yaml", + `select(di==0).spec.containers += {"name": "redis", "image": "redis"}`, + "inserts/tc-09-01-expected.yaml", + }, + { + "inserts/tc-10-00-input-list-insert-new-mapping.yaml", + `select(di==0).spec.containers += {"name": "redis", "image": "redis"}`, + "inserts/tc-10-01-expected.yaml", + }, + + // Removal Scenarios + { + "removals/tc-01-00-input.yaml", + "del(select(di==0).spec.containers[0].securityContext)", + "removals/tc-01-01-expected.yaml", + }, + { + "removals/tc-02-00-input.yaml", + "del(select(di==0).spec.containers[1])", + "removals/tc-02-01-expected.yaml", + }, + { + "removals/tc-03-00-input.yaml", + "del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1])", + "removals/tc-03-01-expected.yaml", + }, + { + "removes/tc-04-00-input.yaml", + `del(select(di==0).spec.containers[0].securityContext) | + del(select(di==1).spec.containers[1])`, + "removes/tc-04-01-expected.yaml", + }, + + // Replace Scenarios + { + "replaces/tc-01-00-input.yaml", + "select(di==0).spec.containers[0].securityContext.runAsRoot |= false", + "replaces/tc-01-01-expected.yaml", + }, + { + "replaces/tc-02-00-input.yaml", + `select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" | + select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW"`, + "replaces/tc-02-01-expected.yaml", + }, + + // Hybrid Scenarios + { + "hybrids/tc-01-00-input.yaml", + `del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`, + "hybrids/tc-01-01-expected.yaml", + }, + { + "hybrids/tc-02-00-input-indented-list.yaml", + `del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`, + "hybrids/tc-02-01-expected.yaml", + }, + { + "hybrids/tc-03-00-input-comments.yaml", + `del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`, + "hybrids/tc-03-01-expected.yaml", + }, + { + "hybrids/tc-04-00-input-separated-keys.yaml", + `del(select(di==0).spec.containers[0].securityContext) | + select(di==0).spec.securityContext.runAsRoot |= false`, + "hybrids/tc-04-01-expected.yaml", + }, } - // make changes to temp file - h, _ := NewFixHandlerMock() - yamlExpression := "select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true" - err = h.applyFixToFile(tempFile.Name(), yamlExpression) - assert.NoError(t, err) + return indentationTestCases +} + +func TestApplyFixKeepsFormatting(t *testing.T) { + testCases := getTestCases() + + for _, tc := range testCases { + t.Run(tc.inputFile, func(t *testing.T) { + getTestDataPath := func(filename string) string { + currentDir, _ := os.Getwd() + currentFile := "testdata/" + filename + return filepath.Join(currentDir, currentFile) + } + + input, _ := os.ReadFile(getTestDataPath(tc.inputFile)) + wantRaw, _ := os.ReadFile(getTestDataPath(tc.expectedFile)) + want := string(wantRaw) + expression := tc.yamlExpression + + h, _ := NewFixHandlerMock() + + got, _ := h.ApplyFixToContent(string(input), expression) + + assert.Equalf( + t, want, got, + "Contents of the fixed file don't match the expectation.\n"+ + "Input file: %s\n\n"+ + "Got: <%s>\n\n"+ + "Want: <%s>", + tc.inputFile, got, want, + ) + }, + ) - // Check temp file contents - b, err = ioutil.ReadFile(tempFile.Name()) - if err != nil { - panic(err) } - assert.Contains(t, string(b), "readOnlyRootFilesystem: true") } func Test_fixPathToValidYamlExpression(t *testing.T) { diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-01-00-input.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-01-00-input.yaml new file mode 100644 index 00000000..f8f5b61a --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-01-00-input.yaml @@ -0,0 +1,19 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: true diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-01-01-expected.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-01-01-expected.yaml new file mode 100644 index 00000000..b74c6cdc --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-01-01-expected.yaml @@ -0,0 +1,19 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-02-00-input-indented-list.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-02-00-input-indented-list.yaml new file mode 100644 index 00000000..8967d5f6 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-02-00-input-indented-list.yaml @@ -0,0 +1,19 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: true diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-02-01-expected.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-02-01-expected.yaml new file mode 100644 index 00000000..7fd9b38b --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-02-01-expected.yaml @@ -0,0 +1,19 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-03-00-input-comments.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-03-00-input-comments.yaml new file mode 100644 index 00000000..ea5c9c51 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-03-00-input-comments.yaml @@ -0,0 +1,21 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + # These are the container comments + containers: + # These are the first containers comments + - name: nginx_container + image: nginx + securityContext: + runAsRoot: true diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-03-01-expected.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-03-01-expected.yaml new file mode 100644 index 00000000..87ef2595 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-03-01-expected.yaml @@ -0,0 +1,21 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + # These are the container comments + containers: + # These are the first containers comments + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-04-00-input-separated-keys.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-04-00-input-separated-keys.yaml new file mode 100644 index 00000000..51951d18 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-04-00-input-separated-keys.yaml @@ -0,0 +1,21 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + + image: nginx + + securityContext: + runAsRoot: true diff --git a/core/pkg/fixhandler/testdata/hybrids/tc-04-01-expected.yaml b/core/pkg/fixhandler/testdata/hybrids/tc-04-01-expected.yaml new file mode 100644 index 00000000..485cfa26 --- /dev/null +++ b/core/pkg/fixhandler/testdata/hybrids/tc-04-01-expected.yaml @@ -0,0 +1,21 @@ +# Fix to Apply: +# REMOVE: +# "del(select(di==0).spec.containers[0].securityContext)" + +# INSERT: +# select(di==0).spec.securityContext.runAsRoot: false + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + + image: nginx + securityContext: + runAsRoot: false + diff --git a/core/pkg/fixhandler/testdata/inserts/tc-01-00-input-mapping-insert-mapping.yaml b/core/pkg/fixhandler/testdata/inserts/tc-01-00-input-mapping-insert-mapping.yaml new file mode 100644 index 00000000..c6be2de9 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-01-00-input-mapping-insert-mapping.yaml @@ -0,0 +1,12 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/inserts/tc-01-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-01-01-expected.yaml new file mode 100644 index 00000000..dbe3c81e --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-01-01-expected.yaml @@ -0,0 +1,14 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + allowPrivilegeEscalation: false diff --git a/core/pkg/fixhandler/testdata/inserts/tc-02-00-input-mapping-insert-mapping-with-list.yaml b/core/pkg/fixhandler/testdata/inserts/tc-02-00-input-mapping-insert-mapping-with-list.yaml new file mode 100644 index 00000000..d88937a4 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-02-00-input-mapping-insert-mapping-with-list.yaml @@ -0,0 +1,11 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx_container + image: nginx \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/inserts/tc-02-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-02-01-expected.yaml new file mode 100644 index 00000000..ba084424 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-02-01-expected.yaml @@ -0,0 +1,15 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + capabilities: + drop: + - NET_RAW \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/inserts/tc-03-00-input-list-append-scalar.yaml b/core/pkg/fixhandler/testdata/inserts/tc-03-00-input-list-append-scalar.yaml new file mode 100644 index 00000000..3d6c873a --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-03-00-input-list-append-scalar.yaml @@ -0,0 +1,15 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["SYS_ADM"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + capabilities: + drop: + - NET_RAW \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/inserts/tc-03-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-03-01-expected.yaml new file mode 100644 index 00000000..e673574d --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-03-01-expected.yaml @@ -0,0 +1,16 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["SYS_ADM"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + capabilities: + drop: + - NET_RAW + - SYS_ADM \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/inserts/tc-04-00-input-multiple-inserts.yaml b/core/pkg/fixhandler/testdata/inserts/tc-04-00-input-multiple-inserts.yaml new file mode 100644 index 00000000..5ef36f1e --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-04-00-input-multiple-inserts.yaml @@ -0,0 +1,47 @@ +# Fixes to Apply: +# 1) select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation = false +# 2) select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] +# 3) select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type = RuntimeDefault +# 4) select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false +# 5) select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: multiple_inserts +spec: + selector: + matchLabels: + app: example_4 + template: + metadata: + labels: + app: example_4 + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + containers: + - name: example_4 + image: nginx + ports: + - containerPort: 3000 + env: + - name: PORT + value: "3000" + resources: + requests: + cpu: 200m + memory: 180Mi + limits: + cpu: 300m + memory: 300Mi + readinessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:3000"] + livenessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:3000"] diff --git a/core/pkg/fixhandler/testdata/inserts/tc-04-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-04-01-expected.yaml new file mode 100644 index 00000000..9827672e --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-04-01-expected.yaml @@ -0,0 +1,57 @@ +# Fixes to Apply: +# 1) select(di==0).spec.template.spec.securityContext.allowPrivilegeEscalation = false +# 2) select(di==0).spec.template.spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] +# 3) select(di==0).spec.template.spec.containers[0].securityContext.seccompProfile.type = RuntimeDefault +# 4) select(di==0).spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation |= false +# 5) select(di==0).spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem |= true + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: multiple_inserts +spec: + selector: + matchLabels: + app: example_4 + template: + metadata: + labels: + app: example_4 + spec: + serviceAccountName: default + terminationGracePeriodSeconds: 5 + containers: + - name: example_4 + image: nginx + ports: + - containerPort: 3000 + env: + - name: PORT + value: "3000" + resources: + requests: + cpu: 200m + memory: 180Mi + limits: + cpu: 300m + memory: 300Mi + readinessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:3000"] + livenessProbe: + initialDelaySeconds: 20 + periodSeconds: 15 + exec: + command: ["/bin/grpc_health_probe", "-addr=:3000"] + securityContext: + capabilities: + drop: + - NET_RAW + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + securityContext: + allowPrivilegeEscalation: false diff --git a/core/pkg/fixhandler/testdata/inserts/tc-05-00-input-comment-blank-line-single-insert.yaml b/core/pkg/fixhandler/testdata/inserts/tc-05-00-input-comment-blank-line-single-insert.yaml new file mode 100644 index 00000000..7e62d9f8 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-05-00-input-comment-blank-line-single-insert.yaml @@ -0,0 +1,16 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + + # Testing if comments are retained as intended + securityContext: + runAsRoot: false \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/inserts/tc-05-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-05-01-expected.yaml new file mode 100644 index 00000000..c9917375 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-05-01-expected.yaml @@ -0,0 +1,18 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + allowPrivilegeEscalation: false + + # Testing if comments are retained as intended + securityContext: + runAsRoot: false \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/inserts/tc-06-00-input-list-append-scalar-oneline.yaml b/core/pkg/fixhandler/testdata/inserts/tc-06-00-input-list-append-scalar-oneline.yaml new file mode 100644 index 00000000..a1638d45 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-06-00-input-list-append-scalar-oneline.yaml @@ -0,0 +1,14 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["SYS_ADM"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: [NET_RAW] \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/inserts/tc-06-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-06-01-expected.yaml new file mode 100644 index 00000000..d64fecd3 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-06-01-expected.yaml @@ -0,0 +1,14 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["SYS_ADM"] + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: [NET_RAW, SYS_ADM] \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/inserts/tc-07-00-input-multiple-documents.yaml b/core/pkg/fixhandler/testdata/inserts/tc-07-00-input-multiple-documents.yaml new file mode 100644 index 00000000..aa6ff7aa --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-07-00-input-multiple-documents.yaml @@ -0,0 +1,27 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + +--- + +# Fix to Apply: +# "select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/inserts/tc-07-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-07-01-expected.yaml new file mode 100644 index 00000000..4085ccbb --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-07-01-expected.yaml @@ -0,0 +1,31 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + allowPrivilegeEscalation: false + +--- + +# Fix to Apply: +# "select(di==1).spec.containers[0].securityContext.allowPrivilegeEscalation |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + allowPrivilegeEscalation: false diff --git a/core/pkg/fixhandler/testdata/inserts/tc-08-00-input-mapping-insert-mapping-indented.yaml b/core/pkg/fixhandler/testdata/inserts/tc-08-00-input-mapping-insert-mapping-indented.yaml new file mode 100644 index 00000000..ac27db64 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-08-00-input-mapping-insert-mapping-indented.yaml @@ -0,0 +1,11 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] + +apiVersion: v1 +kind: Pod +metadata: + name: indented-parent-list-insert-list-value +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/inserts/tc-08-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-08-01-expected.yaml new file mode 100644 index 00000000..46a3acbe --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-08-01-expected.yaml @@ -0,0 +1,15 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop += ["NET_RAW"] + +apiVersion: v1 +kind: Pod +metadata: + name: indented-parent-list-insert-list-value +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + capabilities: + drop: + - NET_RAW diff --git a/core/pkg/fixhandler/testdata/inserts/tc-09-00-input-list-insert-new-mapping-indented.yaml b/core/pkg/fixhandler/testdata/inserts/tc-09-00-input-list-insert-new-mapping-indented.yaml new file mode 100644 index 00000000..416c0c1a --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-09-00-input-list-insert-new-mapping-indented.yaml @@ -0,0 +1,11 @@ +# Fix to Apply: +# select(di==0).spec.containers += {"name": "redis", "image": "redis"} + +apiVersion: v1 +kind: Pod +metadata: + name: indented-parent-list-insert-list-value +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/inserts/tc-09-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-09-01-expected.yaml new file mode 100644 index 00000000..61ce65f7 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-09-01-expected.yaml @@ -0,0 +1,13 @@ +# Fix to Apply: +# select(di==0).spec.containers += {"name": "redis", "image": "redis"} + +apiVersion: v1 +kind: Pod +metadata: + name: indented-parent-list-insert-list-value +spec: + containers: + - name: nginx_container + image: nginx + - name: redis + image: redis diff --git a/core/pkg/fixhandler/testdata/inserts/tc-10-00-input-list-insert-new-mapping.yaml b/core/pkg/fixhandler/testdata/inserts/tc-10-00-input-list-insert-new-mapping.yaml new file mode 100644 index 00000000..827cad1d --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-10-00-input-list-insert-new-mapping.yaml @@ -0,0 +1,11 @@ +# Fix to Apply: +# select(di==0).spec.containers += {"name": "redis", "image": "redis"} + +apiVersion: v1 +kind: Pod +metadata: + name: indented-list-insert-new-object +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/inserts/tc-10-01-expected.yaml b/core/pkg/fixhandler/testdata/inserts/tc-10-01-expected.yaml new file mode 100644 index 00000000..a46f6b75 --- /dev/null +++ b/core/pkg/fixhandler/testdata/inserts/tc-10-01-expected.yaml @@ -0,0 +1,13 @@ +# Fix to Apply: +# select(di==0).spec.containers += {"name": "redis", "image": "redis"} + +apiVersion: v1 +kind: Pod +metadata: + name: indented-list-insert-new-object +spec: + containers: + - name: nginx_container + image: nginx + - name: redis + image: redis diff --git a/core/pkg/fixhandler/testdata/removals/tc-01-00-input.yaml b/core/pkg/fixhandler/testdata/removals/tc-01-00-input.yaml new file mode 100644 index 00000000..579ce100 --- /dev/null +++ b/core/pkg/fixhandler/testdata/removals/tc-01-00-input.yaml @@ -0,0 +1,14 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/removals/tc-01-01-expected.yaml b/core/pkg/fixhandler/testdata/removals/tc-01-01-expected.yaml new file mode 100644 index 00000000..8ecc8724 --- /dev/null +++ b/core/pkg/fixhandler/testdata/removals/tc-01-01-expected.yaml @@ -0,0 +1,12 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/removals/tc-02-00-input.yaml b/core/pkg/fixhandler/testdata/removals/tc-02-00-input.yaml new file mode 100644 index 00000000..29ebfbf9 --- /dev/null +++ b/core/pkg/fixhandler/testdata/removals/tc-02-00-input.yaml @@ -0,0 +1,15 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + + - name: container_with_security_issues + image: image_with_security_issues \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/removals/tc-02-01-expected.yaml b/core/pkg/fixhandler/testdata/removals/tc-02-01-expected.yaml new file mode 100644 index 00000000..56fc35f8 --- /dev/null +++ b/core/pkg/fixhandler/testdata/removals/tc-02-01-expected.yaml @@ -0,0 +1,12 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/removals/tc-03-00-input.yaml b/core/pkg/fixhandler/testdata/removals/tc-03-00-input.yaml new file mode 100644 index 00000000..c53829a8 --- /dev/null +++ b/core/pkg/fixhandler/testdata/removals/tc-03-00-input.yaml @@ -0,0 +1,14 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: ["NET_RAW", "SYS_ADM"] \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/removals/tc-03-01-expected.yaml b/core/pkg/fixhandler/testdata/removals/tc-03-01-expected.yaml new file mode 100644 index 00000000..b0899e2c --- /dev/null +++ b/core/pkg/fixhandler/testdata/removals/tc-03-01-expected.yaml @@ -0,0 +1,14 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext.capabilities.drop[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: ["NET_RAW"] \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/removals/tc-04-00-input.yaml b/core/pkg/fixhandler/testdata/removals/tc-04-00-input.yaml new file mode 100644 index 00000000..c2aafb03 --- /dev/null +++ b/core/pkg/fixhandler/testdata/removals/tc-04-00-input.yaml @@ -0,0 +1,32 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false + +--- + +# Fix to Apply: +# del(select(di==0).spec.containers[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + + - name: container_with_security_issues + image: image_with_security_issues \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/removals/tc-04-01-expected.yaml b/core/pkg/fixhandler/testdata/removals/tc-04-01-expected.yaml new file mode 100644 index 00000000..fd24bbc9 --- /dev/null +++ b/core/pkg/fixhandler/testdata/removals/tc-04-01-expected.yaml @@ -0,0 +1,27 @@ +# Fix to Apply: +# del(select(di==0).spec.containers[0].securityContext) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx + +--- + +# Fix to Apply: +# del(select(di==0).spec.containers[1]) + +apiVersion: v1 +kind: Pod +metadata: + name: remove_example + +spec: + containers: + - name: nginx_container + image: nginx diff --git a/core/pkg/fixhandler/testdata/replaces/tc-01-00-input.yaml b/core/pkg/fixhandler/testdata/replaces/tc-01-00-input.yaml new file mode 100644 index 00000000..335585b2 --- /dev/null +++ b/core/pkg/fixhandler/testdata/replaces/tc-01-00-input.yaml @@ -0,0 +1,14 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.runAsRoot |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: true \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/replaces/tc-01-01-expected.yaml b/core/pkg/fixhandler/testdata/replaces/tc-01-01-expected.yaml new file mode 100644 index 00000000..4a4eda33 --- /dev/null +++ b/core/pkg/fixhandler/testdata/replaces/tc-01-01-expected.yaml @@ -0,0 +1,14 @@ +# Fix to Apply: +# "select(di==0).spec.containers[0].securityContext.runAsRoot |= false" + +apiVersion: v1 +kind: Pod +metadata: + name: insert_to_mapping_node_1 + +spec: + containers: + - name: nginx_container + image: nginx + securityContext: + runAsRoot: false \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/replaces/tc-02-00-input.yaml b/core/pkg/fixhandler/testdata/replaces/tc-02-00-input.yaml new file mode 100644 index 00000000..4a4821c0 --- /dev/null +++ b/core/pkg/fixhandler/testdata/replaces/tc-02-00-input.yaml @@ -0,0 +1,18 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" +# select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW" + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: + - "NET_RAW" + add: ["SYS_ADM"] \ No newline at end of file diff --git a/core/pkg/fixhandler/testdata/replaces/tc-02-01-expected.yaml b/core/pkg/fixhandler/testdata/replaces/tc-02-01-expected.yaml new file mode 100644 index 00000000..4ea9b48c --- /dev/null +++ b/core/pkg/fixhandler/testdata/replaces/tc-02-01-expected.yaml @@ -0,0 +1,18 @@ +# Fix to Apply: +# select(di==0).spec.containers[0].securityContext.capabilities.drop[0] |= "SYS_ADM" +# select(di==0).spec.containers[0].securityContext.capabilities.add[0] |= "NET_RAW" + + +apiVersion: v1 +kind: Pod +metadata: + name: insert_list +spec: + containers: + - name: nginx1 + image: nginx + securityContext: + capabilities: + drop: + - "SYS_ADM" + add: ["NET_RAW"] \ No newline at end of file diff --git a/core/pkg/fixhandler/yamlhandler.go b/core/pkg/fixhandler/yamlhandler.go new file mode 100644 index 00000000..b22f78c8 --- /dev/null +++ b/core/pkg/fixhandler/yamlhandler.go @@ -0,0 +1,286 @@ +package fixhandler + +import ( + "container/list" + "errors" + "fmt" + "io" + "strings" + + "github.com/mikefarah/yq/v4/pkg/yqlib" + + "gopkg.in/yaml.v3" +) + +// decodeDocumentRoots decodes all YAML documents stored in a given `filepath` and returns a slice of their root nodes +func decodeDocumentRoots(yamlAsString string) ([]yaml.Node, error) { + fileReader := strings.NewReader(yamlAsString) + dec := yaml.NewDecoder(fileReader) + + nodes := make([]yaml.Node, 0) + for { + var node yaml.Node + err := dec.Decode(&node) + + nodes = append(nodes, node) + + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("Cannot Decode File as YAML") + + } + } + + return nodes, nil +} + +func getFixedNodes(yamlAsString, yamlExpression string) ([]yaml.Node, error) { + preferences := yqlib.ConfiguredYamlPreferences + preferences.EvaluateTogether = true + decoder := yqlib.NewYamlDecoder(preferences) + + var allDocuments = list.New() + reader := strings.NewReader(yamlAsString) + + fileDocuments, err := readDocuments(reader, decoder) + if err != nil { + return nil, err + } + allDocuments.PushBackList(fileDocuments) + + allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator() + + fixedCandidateNodes, err := allAtOnceEvaluator.EvaluateCandidateNodes(yamlExpression, allDocuments) + + if err != nil { + return nil, fmt.Errorf("Error fixing YAML, %w", err) + } + + fixedNodes := make([]yaml.Node, 0) + var fixedNode *yaml.Node + for fixedCandidateNode := fixedCandidateNodes.Front(); fixedCandidateNode != nil; fixedCandidateNode = fixedCandidateNode.Next() { + fixedNode = fixedCandidateNode.Value.(*yqlib.CandidateNode).Node + fixedNodes = append(fixedNodes, *fixedNode) + } + + return fixedNodes, nil +} + +func flattenWithDFS(node *yaml.Node) *[]nodeInfo { + dfsOrder := make([]nodeInfo, 0) + flattenWithDFSHelper(node, nil, &dfsOrder, 0) + return &dfsOrder +} + +func flattenWithDFSHelper(node *yaml.Node, parent *yaml.Node, dfsOrder *[]nodeInfo, index int) { + dfsNode := nodeInfo{ + node: node, + parent: parent, + index: index, + } + *dfsOrder = append(*dfsOrder, dfsNode) + + for idx, child := range node.Content { + flattenWithDFSHelper(child, node, dfsOrder, idx) + } +} + +func getFixInfo(originalRootNodes, fixedRootNodes []yaml.Node) fileFixInfo { + contentToAdd := make([]contentToAdd, 0) + linesToRemove := make([]linesToRemove, 0) + + for idx := 0; idx < len(fixedRootNodes); idx++ { + originalList := flattenWithDFS(&originalRootNodes[idx]) + fixedList := flattenWithDFS(&fixedRootNodes[idx]) + nodeContentToAdd, nodeLinesToRemove := getFixInfoHelper(*originalList, *fixedList) + contentToAdd = append(contentToAdd, nodeContentToAdd...) + linesToRemove = append(linesToRemove, nodeLinesToRemove...) + } + + return fileFixInfo{ + contentsToAdd: &contentToAdd, + linesToRemove: &linesToRemove, + } +} + +func getFixInfoHelper(originalList, fixedList []nodeInfo) ([]contentToAdd, []linesToRemove) { + + // While obtaining fixedYamlNode, comments and empty lines at the top are ignored. + // This causes a difference in Line numbers across the tree structure. In order to + // counter this, line numbers are adjusted in fixed list. + adjustFixedListLines(&originalList, &fixedList) + + contentToAdd := make([]contentToAdd, 0) + linesToRemove := make([]linesToRemove, 0) + + originalListTracker, fixedListTracker := 0, 0 + + fixInfoMetadata := &fixInfoMetadata{ + originalList: &originalList, + fixedList: &fixedList, + originalListTracker: originalListTracker, + fixedListTracker: fixedListTracker, + contentToAdd: &contentToAdd, + linesToRemove: &linesToRemove, + } + + for originalListTracker < len(originalList) && fixedListTracker < len(fixedList) { + matchNodeResult := matchNodes(originalList[originalListTracker].node, fixedList[fixedListTracker].node) + + fixInfoMetadata.originalListTracker = originalListTracker + fixInfoMetadata.fixedListTracker = fixedListTracker + + switch matchNodeResult { + case sameNodes: + originalListTracker += 1 + fixedListTracker += 1 + + case removedNode: + originalListTracker, fixedListTracker = addLinesToRemove(fixInfoMetadata) + + case insertedNode: + originalListTracker, fixedListTracker = addLinesToInsert(fixInfoMetadata) + + case replacedNode: + originalListTracker, fixedListTracker = updateLinesToReplace(fixInfoMetadata) + } + } + + // Some nodes are still not visited if they are removed at the end of the list + for originalListTracker < len(originalList) { + fixInfoMetadata.originalListTracker = originalListTracker + originalListTracker, _ = addLinesToRemove(fixInfoMetadata) + } + + // Some nodes are still not visited if they are inserted at the end of the list + for fixedListTracker < len(fixedList) { + // Use negative index of last node in original list as a placeholder to determine the last line number later + fixInfoMetadata.originalListTracker = -(len(originalList) - 1) + fixInfoMetadata.fixedListTracker = fixedListTracker + _, fixedListTracker = addLinesToInsert(fixInfoMetadata) + } + + return contentToAdd, linesToRemove + +} + +// Adds the lines to remove and returns the updated originalListTracker +func addLinesToRemove(fixInfoMetadata *fixInfoMetadata) (int, int) { + isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) + + if isOneLine { + // Remove the entire line and replace it with the sequence node in fixed info. This way, + // the original formatting is not lost. + return replaceSingleLineSequence(fixInfoMetadata, line) + } + + currentDFSNode := (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker] + + newOriginalListTracker := updateTracker(fixInfoMetadata.originalList, fixInfoMetadata.originalListTracker) + *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, linesToRemove{ + startLine: currentDFSNode.node.Line, + endLine: getNodeLine(fixInfoMetadata.originalList, newOriginalListTracker), + }) + + return newOriginalListTracker, fixInfoMetadata.fixedListTracker +} + +// Adds the lines to insert and returns the updated fixedListTracker +func addLinesToInsert(fixInfoMetadata *fixInfoMetadata) (int, int) { + + isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) + + if isOneLine { + return replaceSingleLineSequence(fixInfoMetadata, line) + } + + currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] + + lineToInsert := getLineToInsert(fixInfoMetadata) + contentToInsert := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) + + newFixedTracker := updateTracker(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) + + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, contentToAdd{ + line: lineToInsert, + content: contentToInsert, + }) + + return fixInfoMetadata.originalListTracker, newFixedTracker +} + +// Adds the lines to remove and insert and updates the fixedListTracker and originalListTracker +func updateLinesToReplace(fixInfoMetadata *fixInfoMetadata) (int, int) { + + isOneLine, line := isOneLineSequenceNode(fixInfoMetadata.fixedList, fixInfoMetadata.fixedListTracker) + + if isOneLine { + return replaceSingleLineSequence(fixInfoMetadata, line) + } + + currentDFSNode := (*fixInfoMetadata.fixedList)[fixInfoMetadata.fixedListTracker] + + // If only the value node is changed, entire "key-value" pair is replaced + if isValueNodeinMapping(¤tDFSNode) { + fixInfoMetadata.originalListTracker -= 1 + fixInfoMetadata.fixedListTracker -= 1 + } + + addLinesToRemove(fixInfoMetadata) + updatedOriginalTracker, updatedFixedTracker := addLinesToInsert(fixInfoMetadata) + + return updatedOriginalTracker, updatedFixedTracker +} + +func removeNewLinesAtTheEnd(yamlLines []string) []string { + for idx := 1; idx < len(yamlLines); idx++ { + if yamlLines[len(yamlLines)-idx] != "\n" { + yamlLines = yamlLines[:len(yamlLines)-idx+1] + break + } + } + return yamlLines +} + +func getFixedYamlLines(yamlLines []string, fileFixInfo fileFixInfo) (fixedYamlLines []string) { + + // Determining last line requires original yaml lines slice. The placeholder for last line is replaced with the real last line + assignLastLine(fileFixInfo.contentsToAdd, fileFixInfo.linesToRemove, &yamlLines) + + removeLines(fileFixInfo.linesToRemove, &yamlLines) + + fixedYamlLines = make([]string, 0) + lineIdx, lineToAddIdx := 1, 0 + + // Ideally, new node is inserted at line before the next node in DFS order. But, when the previous line contains a + // comment or empty line, we need to insert new nodes before them. + adjustContentLines(fileFixInfo.contentsToAdd, &yamlLines) + + for lineToAddIdx < len(*fileFixInfo.contentsToAdd) { + for lineIdx <= (*fileFixInfo.contentsToAdd)[lineToAddIdx].line { + // Check if the current line is not removed + if yamlLines[lineIdx-1] != "*" { + fixedYamlLines = append(fixedYamlLines, yamlLines[lineIdx-1]) + } + lineIdx += 1 + } + + content := (*fileFixInfo.contentsToAdd)[lineToAddIdx].content + fixedYamlLines = append(fixedYamlLines, content) + + lineToAddIdx += 1 + } + + for lineIdx <= len(yamlLines) { + if yamlLines[lineIdx-1] != "*" { + fixedYamlLines = append(fixedYamlLines, yamlLines[lineIdx-1]) + } + lineIdx += 1 + } + + fixedYamlLines = removeNewLinesAtTheEnd(fixedYamlLines) + + return fixedYamlLines +} diff --git a/core/pkg/fixhandler/yamlhelper.go b/core/pkg/fixhandler/yamlhelper.go new file mode 100644 index 00000000..ad3267ec --- /dev/null +++ b/core/pkg/fixhandler/yamlhelper.go @@ -0,0 +1,406 @@ +package fixhandler + +import ( + "bufio" + "bytes" + "container/list" + "errors" + "fmt" + "io" + "math" + "os" + "strings" + + logger "github.com/kubescape/go-logger" + "github.com/mikefarah/yq/v4/pkg/yqlib" + "gopkg.in/yaml.v3" +) + +type NodeRelation int + +const ( + sameNodes NodeRelation = iota + insertedNode + removedNode + replacedNode +) + +func matchNodes(nodeOne, nodeTwo *yaml.Node) NodeRelation { + + isNewNode := nodeTwo.Line == 0 && nodeTwo.Column == 0 + sameLines := nodeOne.Line == nodeTwo.Line + sameColumns := nodeOne.Column == nodeTwo.Column + + isSameNode := isSameNode(nodeOne, nodeTwo) + + switch { + case isSameNode: + return sameNodes + case isNewNode: + return insertedNode + case sameLines && sameColumns: + return replacedNode + default: + return removedNode + } +} + +func adjustContentLines(contentToAdd *[]contentToAdd, linesSlice *[]string) { + for contentIdx, content := range *contentToAdd { + line := content.line + + // Adjust line numbers such that there are no "empty lines or comment lines of next nodes" before them + for idx := line - 1; idx >= 0; idx-- { + if isEmptyLineOrComment((*linesSlice)[idx]) { + (*contentToAdd)[contentIdx].line -= 1 + } else { + break + } + } + } +} + +func adjustFixedListLines(originalList, fixedList *[]nodeInfo) { + differenceAtTop := (*originalList)[0].node.Line - (*fixedList)[0].node.Line + + if differenceAtTop <= 0 { + return + } + + for _, node := range *fixedList { + // line numbers should not be changed for new nodes. + if node.node.Line != 0 { + node.node.Line += differenceAtTop + } + } + + return + +} + +func enocodeIntoYaml(parentNode *yaml.Node, nodeList *[]nodeInfo, tracker int) (string, error) { + content := make([]*yaml.Node, 0) + currentNode := (*nodeList)[tracker].node + content = append(content, currentNode) + + // Add the value in "key-value" pair to construct if the parent is mapping node + if parentNode.Kind == yaml.MappingNode { + valueNode := (*nodeList)[tracker+1].node + content = append(content, valueNode) + } + + // The parent is added at the top to encode into YAML + parentForContent := yaml.Node{ + Kind: parentNode.Kind, + Content: content, + } + + buf := new(bytes.Buffer) + + encoder := yaml.NewEncoder(buf) + encoder.SetIndent(2) + + errorEncoding := encoder.Encode(parentForContent) + if errorEncoding != nil { + return "", fmt.Errorf("Error debugging node, %v", errorEncoding.Error()) + } + errorClosingEncoder := encoder.Close() + if errorClosingEncoder != nil { + return "", fmt.Errorf("Error closing encoder: %v", errorClosingEncoder.Error()) + } + return fmt.Sprintf(`%v`, buf.String()), nil +} + +func getContent(parentNode *yaml.Node, nodeList *[]nodeInfo, tracker int) string { + content, err := enocodeIntoYaml(parentNode, nodeList, tracker) + if err != nil { + logger.L().Fatal("Cannot Encode into YAML") + } + + indentationSpaces := parentNode.Column - 1 + + content = indentContent(content, indentationSpaces) + + return strings.TrimSuffix(content, "\n") +} + +func indentContent(content string, indentationSpaces int) string { + indentedContent := "" + indentSpaces := strings.Repeat(" ", indentationSpaces) + + scanner := bufio.NewScanner(strings.NewReader(content)) + for scanner.Scan() { + line := scanner.Text() + indentedContent += (indentSpaces + line + "\n") + } + return indentedContent +} + +func getLineToInsert(fixInfoMetadata *fixInfoMetadata) int { + var lineToInsert int + // Check if lineToInsert is last line + if fixInfoMetadata.originalListTracker < 0 { + originalListTracker := int(math.Abs(float64(fixInfoMetadata.originalListTracker))) + // Storing the negative value of line of last node as a placeholder to determine the last line later. + lineToInsert = -(*fixInfoMetadata.originalList)[originalListTracker].node.Line + } else { + lineToInsert = (*fixInfoMetadata.originalList)[fixInfoMetadata.originalListTracker].node.Line - 1 + } + return lineToInsert +} + +func assignLastLine(contentsToAdd *[]contentToAdd, linesToRemove *[]linesToRemove, linesSlice *[]string) { + for idx, contentToAdd := range *contentsToAdd { + if contentToAdd.line < 0 { + currentLine := int(math.Abs(float64(contentToAdd.line))) + (*contentsToAdd)[idx].line, _ = getLastLineOfResource(linesSlice, currentLine) + } + } + + for idx, lineToRemove := range *linesToRemove { + if lineToRemove.endLine < 0 { + endLine, _ := getLastLineOfResource(linesSlice, lineToRemove.startLine) + (*linesToRemove)[idx].endLine = endLine + } + } +} + +func getLastLineOfResource(linesSlice *[]string, currentLine int) (int, error) { + // Get lastlines of all resources... + lastLinesOfResources := make([]int, 0) + for lineNumber, lineContent := range *linesSlice { + if lineContent == "---" { + for lastLine := lineNumber - 1; lastLine >= 0; lastLine-- { + if !isEmptyLineOrComment((*linesSlice)[lastLine]) { + lastLinesOfResources = append(lastLinesOfResources, lastLine+1) + break + } + } + } + } + + lastLine := len(*linesSlice) + for lastLine >= 0 { + if !isEmptyLineOrComment((*linesSlice)[lastLine-1]) { + lastLinesOfResources = append(lastLinesOfResources, lastLine) + break + } else { + lastLine-- + } + } + + // Get last line of the resource we need + for _, endLine := range lastLinesOfResources { + if currentLine <= endLine { + return endLine, nil + } + } + + return 0, fmt.Errorf("Provided line is greater than the length of YAML file") +} + +func getNodeLine(nodeList *[]nodeInfo, tracker int) int { + if tracker < len(*nodeList) { + return (*nodeList)[tracker].node.Line + } else { + return -1 + } +} + +// Checks if the node is value node in "key-value" pairs of mapping node +func isValueNodeinMapping(node *nodeInfo) bool { + if node.parent.Kind == yaml.MappingNode && node.index%2 != 0 { + return true + } + return false +} + +// Checks if the node is part of single line sequence node and returns the line +func isOneLineSequenceNode(list *[]nodeInfo, currentTracker int) (bool, int) { + parentNode := (*list)[currentTracker].parent + if parentNode.Kind != yaml.SequenceNode { + return false, -1 + } + + var currentNode, prevNode nodeInfo + currentTracker -= 1 + + for (*list)[currentTracker].node != parentNode { + currentNode = (*list)[currentTracker] + prevNode = (*list)[currentTracker-1] + + if currentNode.node.Line != prevNode.node.Line { + return false, -1 + } + currentTracker -= 1 + } + + parentNodeInfo := (*list)[currentTracker] + + if parentNodeInfo.parent.Kind == yaml.MappingNode { + keyNodeInfo := (*list)[currentTracker-1] + if keyNodeInfo.node.Line == parentNode.Line { + return true, parentNode.Line + } else { + return false, -1 + } + } else { + if parentNodeInfo.parent.Line == parentNode.Line { + return true, parentNode.Line + } else { + return false, -1 + } + } +} + +// Checks if nodes are of same kind, value, line and column +func isSameNode(nodeOne, nodeTwo *yaml.Node) bool { + sameLines := nodeOne.Line == nodeTwo.Line + sameColumns := nodeOne.Column == nodeTwo.Column + sameKinds := nodeOne.Kind == nodeTwo.Kind + sameValues := nodeOne.Value == nodeTwo.Value + + return sameKinds && sameValues && sameLines && sameColumns +} + +// Checks if the line is empty or a comment +func isEmptyLineOrComment(lineContent string) bool { + lineContent = strings.TrimSpace(lineContent) + if lineContent == "" { + return true + } else if lineContent[0:1] == "#" { + return true + } + return false +} + +func readDocuments(reader io.Reader, decoder yqlib.Decoder) (*list.List, error) { + err := decoder.Init(reader) + if err != nil { + return nil, fmt.Errorf("Error Initializing the decoder, %w", err) + } + inputList := list.New() + + var currentIndex uint + + for { + candidateNode, errorReading := decoder.Decode() + + if errors.Is(errorReading, io.EOF) { + switch reader := reader.(type) { + case *os.File: + safelyCloseFile(reader) + } + return inputList, nil + } else if errorReading != nil { + return nil, fmt.Errorf("Error Decoding YAML file, %w", errorReading) + } + + candidateNode.Document = currentIndex + candidateNode.EvaluateTogether = true + + inputList.PushBack(candidateNode) + + currentIndex = currentIndex + 1 + } +} + +func safelyCloseFile(file *os.File) { + err := file.Close() + if err != nil { + logger.L().Error("Error Closing File") + } +} + +// Remove the entire line and replace it with the sequence node in fixed info. This way, +// the original formatting is lost. +func replaceSingleLineSequence(fixInfoMetadata *fixInfoMetadata, line int) (int, int) { + originalListTracker := getFirstNodeInLine(fixInfoMetadata.originalList, line) + fixedListTracker := getFirstNodeInLine(fixInfoMetadata.fixedList, line) + + currentDFSNode := (*fixInfoMetadata.fixedList)[fixedListTracker] + contentToInsert := getContent(currentDFSNode.parent, fixInfoMetadata.fixedList, fixedListTracker) + + // Remove the Single line + *fixInfoMetadata.linesToRemove = append(*fixInfoMetadata.linesToRemove, linesToRemove{ + startLine: line, + endLine: line, + }) + + // Encode entire Sequence Node and Insert + *fixInfoMetadata.contentToAdd = append(*fixInfoMetadata.contentToAdd, contentToAdd{ + line: line, + content: contentToInsert, + }) + + originalListTracker = updateTracker(fixInfoMetadata.originalList, originalListTracker) + fixedListTracker = updateTracker(fixInfoMetadata.fixedList, fixedListTracker) + + return originalListTracker, fixedListTracker +} + +// Returns the first node in the given line that is not mapping node +func getFirstNodeInLine(list *[]nodeInfo, line int) int { + tracker := 0 + + currentNode := (*list)[tracker].node + for currentNode.Line != line || currentNode.Kind == yaml.MappingNode { + tracker += 1 + currentNode = (*list)[tracker].node + } + + return tracker +} + +// To not mess with the line number while inserting, removed lines are not deleted but replaced with "*" +func removeLines(linesToRemove *[]linesToRemove, linesSlice *[]string) { + var startLine, endLine int + for _, lineToRemove := range *linesToRemove { + startLine = lineToRemove.startLine - 1 + endLine = lineToRemove.endLine - 1 + + for line := startLine; line <= endLine; line++ { + lineContent := (*linesSlice)[line] + // When determining the endLine, empty lines and comments which are not intended to be removed are included. + // To deal with that, we need to refrain from removing empty lines and comments + if isEmptyLineOrComment(lineContent) { + break + } + (*linesSlice)[line] = "*" + } + } +} + +// Skips the current node including it's children in DFS order and returns the new tracker. +func skipCurrentNode(node *yaml.Node, currentTracker int) int { + updatedTracker := currentTracker + getChildrenCount(node) + return updatedTracker +} + +func getChildrenCount(node *yaml.Node) int { + totalChildren := 1 + for _, child := range node.Content { + totalChildren += getChildrenCount(child) + } + return totalChildren +} + +// The current node along with it's children is skipped and the tracker is moved to next sibling +// of current node. If parent is mapping node, "value" in "key-value" pairs is also skipped. +func updateTracker(nodeList *[]nodeInfo, tracker int) int { + currentNode := (*nodeList)[tracker] + var updatedTracker int + + if currentNode.parent.Kind == yaml.MappingNode { + valueNode := (*nodeList)[tracker+1] + updatedTracker = skipCurrentNode(valueNode.node, tracker+1) + } else { + updatedTracker = skipCurrentNode(currentNode.node, tracker) + } + + return updatedTracker +} + +func getStringFromSlice(yamlLines []string) (fixedYamlString string) { + return strings.Join(yamlLines, "\n") +} diff --git a/core/pkg/resourcehandler/filesloader.go b/core/pkg/resourcehandler/filesloader.go index 933e392b..600a662c 100644 --- a/core/pkg/resourcehandler/filesloader.go +++ b/core/pkg/resourcehandler/filesloader.go @@ -103,6 +103,8 @@ func getResourcesFromPath(path string) (map[string]reporthandling.Source, []work gitRepo, err := cautils.NewLocalGitRepository(path) if err == nil && gitRepo != nil { repoRoot, _ = gitRepo.GetRootDir() + } else { + repoRoot, _ = filepath.Abs(path) } // load resource from local file system @@ -141,7 +143,7 @@ func getResourcesFromPath(path string) (map[string]reporthandling.Source, []work } workloadSource := reporthandling.Source{ - RelativePath: source, + RelativePath: relSource, FileType: filetype, LastCommit: lastCommit, }