update changelog and docs (#800)

* update changelog and docs

* add 7.0.1

* fix version

.circleci/config.yml

@@ -2,6 +2,7 @@ version: 2.1
 
 orbs:
   rok8s: fairwinds/rok8s-scripts@11
+  oss-docs: fairwinds/oss-docs@0
 
 executors:
   vm:
@@ -205,48 +206,6 @@ jobs:
       - setup_remote_docker
       - *set_environment_variables
       - *docker_build_and_push
-  publish_docs:
-    docker:
-      - image: cimg/node:15.5.1
-    steps:
-      - checkout
-      - run:
-          name: Build Docs Site
-          command: |
-            set -e
-            cd ./docs
-            npm install
-            npm run check-links
-            npm run build
-      - run:
-          name: Install Tools
-          command: |
-            cd /tmp
-            echo "Installing AWS CLI"
-            curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-            unzip awscliv2.zip
-            sudo ./aws/install
-
-            echo "Installing Hashicorp Vault"
-            curl -LO https://releases.hashicorp.com/vault/1.9.3/vault_1.9.3_linux_amd64.zip
-            unzip vault_1.9.3_linux_amd64.zip
-            sudo mv vault /usr/bin/vault
-            sudo chmod +x /usr/bin/vault
-            vault --version
-
-            echo "Installing yq"
-            curl -LO https://github.com/mikefarah/yq/releases/download/v4.16.2/yq_linux_amd64.tar.gz
-            tar -zxvf yq_linux_amd64.tar.gz
-            sudo mv yq_linux_amd64 /usr/bin/yq
-            sudo chmod +x /usr/bin/yq
-            yq --version
-      - rok8s/get_vault_env:
-          vault_path: repo/polaris/env
-      - run:
-          name: Publish Docs Site to S3
-          command: |
-            cd ./dist
-            aws s3 sync ./ s3://polaris.docs.fairwinds.com --delete
 workflows:
   version: 2
 
@@ -294,7 +253,8 @@ workflows:
               ignore: /.*/
             tags:
               ignore: /^testing-.*/
-      - publish_docs:
+      - oss-docs/publish-docs:
+          repository: polaris
           filters:
             branches:
               ignore: /.*/

checks/cpuLimitsMissing.yaml

@@ -26,12 +26,9 @@ schema:
               not:
                 const: ''
 mutations:
-  - op: add
-    path: /resources/limits
-    value: {}
   - op: add
     path: /resources/limits/cpu
     value: 100m
 comments:
   - find: "cpu: 100m"
-    comment: "TODO: Set this to the amount of CPU you want to reserve for your workload"
+    comment: "TODO: Set this to the maximum amount of CPU you want your workload to use"

checks/cpuRequestsMissing.yaml

@@ -26,12 +26,9 @@ schema:
               not:
                 const: ''
 mutations:
-  - op: add
-    path: /resources/requests
-    value: {}
   - op: add
     path: /resources/requests/cpu
     value: 100m
 comments:
   - find: "cpu: 100m"
-    comment: "TODO: Set this to the amount of CPU you want to reserve for your workload"
+    comment: "TODO: Set this to the amount of CPU you want to reserve for your workload"

checks/dangerousCapabilities.yaml

@@ -24,3 +24,8 @@ schema:
                 - not:
                     contains:
                       pattern: '^(?i)NET_ADMIN$'
+
+
+mutations:
+  - op: remove
+    path: /securityContext/capabilities

checks/insecureCapabilities.yaml

@@ -54,3 +54,9 @@ schema:
                     pattern: '^(?i)KILL$'
                 - contains:
                     pattern: '^(?i)AUDIT_WRITE$'
+mutations:
+  - op: remove
+    path: /securityContext/capabilities
+  - op: add
+    path: /securityContext/capabilities
+    value: {"drop": ["ALL"]}

checks/memoryLimitsMissing.yaml

@@ -26,12 +26,9 @@ schema:
               not:
                 const: ''
 mutations:
-  - op: add
-    path: /resources/limits
-    value: {}
   - op: add
     path: /resources/limits/memory
     value: "512Mi"
 comments:
   - find: "memory: 512Mi"
-    comment: "TODO: Set this to the amount of Memory you want to reserve for your workload"
+    comment: "TODO: Set this to the maximum amount of memory you want your workload to use"

checks/memoryRequestsMissing.yaml

@@ -26,12 +26,9 @@ schema:
               not:
                 const: ''
 mutations:
-  - op: add
-    path: /resources/requests
-    value: {}
   - op: add
     path: /resources/requests/memory
     value: "512Mi"
 comments:
   - find: "memory: 512Mi"
-    comment: "TODO: Set this to the amount of Memory you want to reserve for your workload"
+    comment: "TODO: Set this to the amount of Memory you want to reserve for your workload"

checks/notReadOnlyRootFilesystem.yaml

@@ -40,4 +40,9 @@ schema:
           - securityContext
           properties:
             securityContext:
-              $ref: "#/definitions/goodSecurityContext"
+              $ref: "#/definitions/goodSecurityContext"
+mutations:
+  - op: add
+    path: /securityContext/readOnlyRootFilesystem
+    value: true
+

checks/runAsRootAllowed.yaml

@@ -51,6 +51,6 @@ schema:
             securityContext:
               $ref: "#/definitions/goodSecurityContext"
 mutations:
-  - op: replace
+  - op: add
     path: /securityContext/runAsNonRoot
-    value: true
+    value: true

cmd/polaris/fix.go

@@ -33,11 +33,16 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
-var filesPath string
+var (
+	filesPath   string
+	checksToFix []string
+	fixAll      bool
+)
 
 func init() {
 	rootCmd.AddCommand(fixCommand)
 	fixCommand.PersistentFlags().StringVar(&filesPath, "files-path", "", "mutate and fix one or more YAML files in a specified folder")
+	fixCommand.PersistentFlags().StringSliceVar(&checksToFix, "checks", []string{}, "Optional flag to specify specific checks to fix eg. checks=hostIPCSet,hostPIDSet and checks=all applies fix to all defined checks mutations")
 }
 
 var fixCommand = &cobra.Command{
@@ -73,12 +78,24 @@ var fixCommand = &cobra.Command{
 		}
 		var contentStr string
 		isFirstResource := true
+
+		if len(checksToFix) > 0 {
+			if len(checksToFix) == 1 && checksToFix[0] == "all" {
+				allchecks := []string{}
+				for key := range config.Checks {
+					allchecks = append(allchecks, key)
+				}
+				config.Mutations = allchecks
+			} else {
+				config.Mutations = checksToFix
+			}
+		}
+
 		for _, fullFilePath := range yamlFiles {
 
 			yamlFile, err := ioutil.ReadFile(fullFilePath)
 			if err != nil {
-				logrus.Errorf("Error reading file with file path %s: %v", fullFilePath, err)
+				logrus.Fatalf("Error reading file with file path %s: %v", fullFilePath, err)
-				os.Exit(1)
 			}
 
 			dec := yamlV3.NewDecoder(bytes.NewReader(yamlFile))
@@ -95,19 +112,16 @@ var fixCommand = &cobra.Command{
 					break
 				}
 				if err != nil {
-					logrus.Errorf("Error decoding data for file with file path %s: %v", fullFilePath, err)
+					logrus.Fatalf("Error decoding data for file with file path %s: %v", fullFilePath, err)
-					os.Exit(1)
 				}
 				yamlContent, err := yamlV3.Marshal(data)
 				if err != nil {
-					logrus.Errorf("Error marshalling %s: %v", fullFilePath, err)
+					logrus.Fatalf("Error marshalling %s: %v", fullFilePath, err)
-					os.Exit(1)
 				}
 				kubeResources := kube.CreateResourceProviderFromYaml(string(yamlContent))
 				results, err := validator.ApplyAllSchemaChecksToResourceProvider(&config, kubeResources)
 				if err != nil {
-					logrus.Errorf("Error applying schema check to the resources %s: %v", fullFilePath, err)
+					logrus.Fatalf("Error applying schema check to the resources %s: %v", fullFilePath, err)
-					os.Exit(1)
 				}
 				comments, allMutations := mutation.GetMutationsAndCommentsFromResults(results)
 				updatedYamlContent := string(yamlContent)
@@ -142,8 +156,7 @@ var fixCommand = &cobra.Command{
 			if contentStr != "" {
 				err = ioutil.WriteFile(fullFilePath, []byte(contentStr), 0644)
 				if err != nil {
-					logrus.Errorf("Error writing output to file: %v", err)
+					logrus.Fatalf("Error writing output to file: %v", err)
-					os.Exit(1)
 				}
 			}
 		}

cmd/polaris/webhook.go

@@ -29,11 +29,15 @@ import (
 
 var webhookPort int
 var disableWebhookConfigInstaller bool
+var enableMutations bool
+var enableValidations bool
 
 func init() {
 	rootCmd.AddCommand(webhookCmd)
 	webhookCmd.PersistentFlags().IntVarP(&webhookPort, "port", "p", 9876, "Port for the dashboard webserver.")
-	webhookCmd.PersistentFlags().BoolVar(&disableWebhookConfigInstaller, "disable-webhook-config-installer", false, "disable the installer in the webhook server, so it won't install webhook configuration resources during bootstrapping.")
+	webhookCmd.PersistentFlags().BoolVar(&disableWebhookConfigInstaller, "disable-webhook-config-installer", false, "Disable the installer in the webhook server, so it won't install webhook configuration resources during bootstrapping.")
+	webhookCmd.PersistentFlags().BoolVar(&enableValidations, "validate", true, "Enable the validating webhook to reject workloads with issues")
+	webhookCmd.PersistentFlags().BoolVar(&enableMutations, "mutate", false, "Enable the mutating webhook to modify workloads with issues")
 }
 
 var webhookCmd = &cobra.Command{
@@ -61,11 +65,17 @@ var webhookCmd = &cobra.Command{
 		server.CertName = "tls.crt"
 		server.KeyName = "tls.key"
 
-		// Iterate all the configurations supported controllers to scan and register them for webhooks
+		if !enableMutations && !enableValidations {
-		// Should only register controllers that are configured to be scanned
+			logrus.Errorf("One of --mutate or --validate must be set to true")
-		fwebhook.NewValidateWebhook(mgr, fwebhook.Validator{Config: config, Client: mgr.GetClient()})
+			os.Exit(1)
-		fwebhook.NewMutateWebhook(mgr, fwebhook.Mutator{Config: config, Client: mgr.GetClient()})
+		}
 
+		if enableValidations {
+			fwebhook.NewValidateWebhook(mgr, fwebhook.Validator{Config: config, Client: mgr.GetClient()})
+		}
+		if enableMutations {
+			fwebhook.NewMutateWebhook(mgr, fwebhook.Mutator{Config: config, Client: mgr.GetClient()})
+		}
 		logrus.Infof("Polaris webhook server listening on port %d", webhookPort)
 		if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
 			logrus.Errorf("Error starting manager: %v", err)

docs/admission-controller.md

@@ -45,3 +45,17 @@ output unless we are rejecting a workload altogether.
 This means that any checks with a severity of `warning` will still pass webhook validation,
 and the only evidence of that warning will either be in the Polaris dashboard or the
 Polaris webhook logs. This will change in a future version of Kubernetes.
+
+## Mutating Webhook
+By default, the Admission Controller is just pass/fail, but
+Polaris can also operate as a mutating webhook for many of the issues it checks for.
+This means Polaris will remediate the issue it finds, rather than rejecting
+the deployment.
+
+To enable the mutating webhook, add `--set webhook.mutate=true` to your
+Helm instlallation command.
+
+By default, the only mutation enabled is `pullPolicyNotAlways`. If you'd like to
+enable other mutations, you can set the `webhook.mutations` flag.
+
+

docs/changelog.md

@@ -5,6 +5,18 @@ meta:
     content: "Fairwinds Polaris | Changelog"
 
 ---
+## 7.0.1
+* Documentation updates
+
+## 7.0.0
+* Better support for `polaris fix`
+* `target: Pod` is now `target: PodSpec` (to differentiate naked Pods from Controllers)
+
+## 6.0.0
+* Preliminary support for `polaris fix` command
+* Changes to how Pod owners are determined
+* Removed YAML manifests from the `deploy/` directory - Helm is now the default install mechanism
+
 ## 5.1.0
 * Support `--context` flag for kubecontext
 * Treat core Kubernetes controllers (`Deployments`, `StatefulSets`, `DaemonSets`, `CronJobs`) as top-level objects, instead of following owner refs

docs/customization/custom-checks.md

@@ -46,7 +46,7 @@ check ID. Note that you'll also have to set its severity in the `checks` section
   * `Controller`, to check _any_ resource that creates Pods (e.g. Deployments, CronJobs, StatefulSets), as well as naked Pods
   * `PodSpec`, same as `Controller`, but the schema applies to the Pod spec rather than the top-level controller
   * `Container` same as `Controller`, but the schema applies to all Container specs rather than the top-level controller
-* `controllers` - if `target` is `Controller`, `Pod` or `Container`, you can use this to change which types of controllers are checked
+* `controllers` - if `target` is `Controller`, `PodSpec` or `Container`, you can use this to change which types of controllers are checked
 * `controllers.include` - _only_ check these controllers
 * `controllers.exclude` - check all controllers except these
 * `containers` - if `target` is `Container`, you can use this to decide if `initContainers`, `containers`, or both should be checked

docs/infrastructure-as-code.md

@@ -25,6 +25,32 @@ brew install FairwindsOps/tap/polaris
 polaris version
 ```
 
+## Checking Infrastructure as Code files
+You can audit Kubernetes YAML files by running:
+```bash
+polaris audit --audit-path ./deploy/ --format=pretty
+```
+This will print out any issues Polaris finds in your manifests.
+
+Polaris can only check raw YAML manifests. If you'd like to check a Helm template,
+you can run `helm template` to generate a manifest that Polaris can check.
+
+## Fixing Issues
+Polaris can automatically fix many of the issues it finds. For example, you can run
+```bash
+polaris fix --files-path ./deploy/ --checks=all
+```
+
+to fix any issues inside the `deploy` directory. Polaris may leave
+comments next to some changes (e.g. liveness and readiness probes) prompting
+the user to set them to something more appropriate given the context of their
+application.
+
+Note that not all issues can be automatically fixed.
+
+Currently only raw YAML manifests can be mutated. Helm charts etc.
+still need to be changed manually.
+
 ## Running in a CI pipeline
 
 ### Set minimum score for an exit code

examples/config.yaml

@@ -27,7 +27,10 @@ checks:
   hostNetworkSet: danger
   hostPortSet: warning
   tlsSettingsMissing: warning
- 
+
+mutations:
+  - pullPolicyNotAlways
+
 exemptions:
   - namespace: kube-system
     controllerNames:

fairwinds-insights.yaml

@@ -1,5 +1,5 @@
 options:
-  organization: fairwinds-opensource
+  organization: fairwinds-production
   baseBranch: master
 
 images:

main.go

@@ -18,7 +18,7 @@ import (
 	cmd "github.com/fairwindsops/polaris/cmd/polaris"
 )
 
-const (
+var (
 	// Version is set during build
 	Version = "development"
 	// Commit is set during build

pkg/mutation/mutate.go

@@ -27,7 +27,10 @@ func ApplyAllSchemaMutations(conf *config.Configuration, resourceProvider *kube.
 	if err != nil {
 		return resource, err
 	}
-	jsonByte, err = patch.Apply(resByte)
+	jsonByte, err = patch.ApplyWithOptions(resByte, &jsonpatchV5.ApplyOptions{
+		AllowMissingPathOnRemove: true,
+		EnsurePathExistsOnAdd: true,
+	})
 	if err != nil {
 		return resource, err
 	}

pkg/validator/schema.go

@@ -325,7 +325,7 @@ func applySchemaCheck(conf *config.Configuration, checkID string, test schemaTes
 	}
 	result := makeResult(conf, check, passes, issues)
 	if !passes {
-		if funk.Contains(conf.Mutations, checkID) {
+		if funk.Contains(conf.Mutations, checkID) && len(check.Mutations) > 0 {
 			mutations := funk.Map(check.Mutations, func(mutation jsonpatch.Operation) jsonpatch.Operation {
 				mutationCopy := deepCopyMutation(mutation)
 				mutationCopy.Path = prefix + mutationCopy.Path

pkg/webhook/mutate.go

@@ -44,7 +44,7 @@ func NewMutateWebhook(mgr manager.Manager, mutator Mutator) {
 }
 
 func (m *Mutator) mutate(req admission.Request) ([]jsonpatch.Operation, error) {
-	results, err := getValidateResults(req.AdmissionRequest.Kind.Kind, m.decoder, req, m.Config)
+	results, err := GetValidatedResults(req.AdmissionRequest.Kind.Kind, m.decoder, req, m.Config)
 	if err != nil {
 		return nil, err
 	}

pkg/webhook/webhook.go

@@ -55,10 +55,11 @@ func NewValidateWebhook(mgr manager.Manager, validator Validator) {
 }
 
 func (v *Validator) handleInternal(req admission.Request) (*validator.Result, error) {
-	return getValidateResults(req.AdmissionRequest.Kind.Kind, v.decoder, req, v.Config)
+	return GetValidatedResults(req.AdmissionRequest.Kind.Kind, v.decoder, req, v.Config)
 }
 
-func getValidateResults(kind string, decoder *admission.Decoder, req admission.Request, config config.Configuration) (*validator.Result, error) {
+// GetValidatedResults returns the validated results.
+func GetValidatedResults(kind string, decoder *admission.Decoder, req admission.Request, config config.Configuration) (*validator.Result, error) {
 	var controller kube.GenericResource
 	var err error
 	if kind == "Pod" {

test/checks/cpuLimitsMissing/mutated.yaml

@@ -10,5 +10,5 @@ spec:
     name: nginx
     resources:
       limits:
-        cpu: 100m #TODO: Set this to the amount of CPU you want to reserve for your workload
+        cpu: 100m #TODO: Set this to the maximum amount of CPU you want your workload to use
         memory: 128Mi

test/checks/dangerousCapabilities/failure.yaml

@@ -6,9 +6,9 @@ metadata:
     app.kubernetes.io/name: nginx
 spec:
   containers:
-  - name: nginx 
+  - name: nginx
     image: nginx
     securityContext:
       capabilities:
         add:
-        - NET_ADMIN
+        - NET_ADMIN

test/checks/dangerousCapabilities/mutated.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/name: nginx
+  name: nginx
+spec:
+  containers:
+  - image: nginx
+    name: nginx
+    resources: {}
+    securityContext: {}

test/checks/insecureCapabilities/mutated.drop-most.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    env: test
+  name: nginx
+spec:
+  containers:
+  - image: nginx
+    name: nginx
+    resources: {}
+    securityContext:
+      capabilities:
+        drop:
+        - ALL

test/checks/insecureCapabilities/mutated.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    env: test
+  name: nginx
+spec:
+  containers:
+  - image: nginx
+    name: nginx
+    resources: {}
+    securityContext:
+      capabilities:
+        drop:
+        - ALL

test/checks/memoryLimitsMissing/failure.cpu.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx
+  labels:
+    app.kubernetes.io/name: nginx
+spec:
+  containers:
+  - name: nginx
+    image: nginx
+    resources:
+      limits:
+        cpu: 100m

test/checks/memoryLimitsMissing/mutated.cpu.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/name: nginx
+  name: nginx
+spec:
+  containers:
+  - image: nginx
+    name: nginx
+    resources:
+      limits:
+        cpu: 100m
+        memory: 512Mi #TODO: Set this to the maximum amount of memory you want your workload to use

test/checks/memoryLimitsMissing/mutated.yaml

@@ -10,4 +10,4 @@ spec:
     name: nginx
     resources:
       limits:
-        memory: 512Mi #TODO: Set this to the amount of Memory you want to reserve for your workload
+        memory: 512Mi #TODO: Set this to the maximum amount of memory you want your workload to use

test/mutation_test.go

@@ -33,14 +33,18 @@ func TestMutations(t *testing.T) {
 	c, err := config.Parse([]byte(configYaml))
 	assert.NoError(t, err)
 	assert.Len(t, c.Mutations, 0)
-	mutations := []string{"hostIPCSet", "pullPolicyNotAlways", "hostPIDSet", "hostNetworkSet", "deploymentMissingReplicas", "runAsRootAllowed", "cpuRequestsMissing", "cpuLimitsMissing", "memoryRequestsMissing", "memoryLimitsMissing", "livenessProbeMissing", "readinessProbeMissing"}
+	for mutationStr := range mutationTestCasesMap {
-	for _, mutationStr := range mutations {
+		if len(mutationTestCasesMap[mutationStr]) == 0 {
-		for _, tc := range failureTestCasesMap[mutationStr] {
+			panic("No test cases found for " + mutationStr)
+		}
+		for _, tc := range mutationTestCasesMap[mutationStr] {
 			newConfig := c
 			key := fmt.Sprintf("%s/%s", tc.check, strings.ReplaceAll(tc.filename, "failure", "mutated"))
 			mutatedYamlContent, ok := mutatedYamlContentMap[key]
 			assert.True(t, ok)
 			assert.Len(t, tc.resources.Resources, 1)
+			newConfig.Checks = map[string]config.Severity{}
+			newConfig.Checks[mutationStr] = config.SeverityDanger
 			newConfig.Mutations = []string{mutationStr}
 			results, err := validator.ApplyAllSchemaChecksToResourceProvider(&newConfig, tc.resources)
 			assert.NoError(t, err)
@@ -56,7 +60,7 @@ func TestMutations(t *testing.T) {
 				yamlContent, err := yaml.JSONToYAML(mutated.OriginalObjectJSON)
 				assert.NoError(t, err)
 				contentStr := mutation.UpdateMutatedContentWithComments(string(yamlContent), comments)
-				assert.EqualValues(t, mutatedYamlContent, contentStr)
+				assert.EqualValues(t, mutatedYamlContent, contentStr, "Mutation test case for " + tc.check + "/" + tc.filename + " failed")
 			}
 		}
 	}

test/schema_test.go

@@ -41,9 +41,10 @@ type testCase struct {
 }
 
 var mutatedYamlContentMap = map[string]string{}
-var failureTestCasesMap = map[string][]testCase{}
+var mutationTestCasesMap = map[string][]testCase{}
 
 func init() {
+	checkToTest := os.Getenv("POLARIS_CHECK_TEST")
 	_, baseDir, _, _ := runtime.Caller(0)
 	baseDir = filepath.Dir(baseDir) + "/checks"
 	dirs, err := ioutil.ReadDir(baseDir)
@@ -52,6 +53,9 @@ func init() {
 	}
 	for _, dir := range dirs {
 		check := dir.Name()
+		if checkToTest != "" && checkToTest != check {
+			continue
+		}
 		checkDir := baseDir + "/" + check
 		cases, err := ioutil.ReadDir(checkDir)
 		if err != nil {
@@ -76,10 +80,19 @@ func init() {
 			if tc.Name() == "check.yaml" {
 				continue
 			}
-			resources, err := kube.CreateResourceProviderFromPath(checkDir + "/" + tc.Name())
+			resourceFilename := strings.Replace(tc.Name(), "mutated", "failure", -1)
+
+			resources, err := kube.CreateResourceProviderFromPath(checkDir + "/" + resourceFilename)
 			if err != nil {
 				panic(err)
 			}
+			testcase := testCase{
+				filename:  tc.Name(),
+				check:     check,
+				resources: resources,
+				failure:   strings.Contains(resourceFilename, "failure"),
+				config:    c,
+			}
 
 			if strings.Contains(tc.Name(), "mutated") {
 				yamlContent, err := os.ReadFile(checkDir + "/" + tc.Name())
@@ -88,24 +101,14 @@ func init() {
 				}
 				key := fmt.Sprintf("%s/%s", check, tc.Name())
 				mutatedYamlContentMap[key] = string(yamlContent)
-			} else {
+				testCases, ok := mutationTestCasesMap[check]
-				testcase := testCase{
+				if !ok {
-					filename:  tc.Name(),
+					testCases = []testCase{}
-					check:     check,
-					resources: resources,
-					failure:   strings.Contains(tc.Name(), "failure"),
-					config:    c,
 				}
 				testCases = append(testCases, testcase)
-
+				mutationTestCasesMap[check] = testCases
-				if strings.Contains(tc.Name(), "mutated") {
+			} else {
-					testCases, ok := failureTestCasesMap[check]
+				testCases = append(testCases, testcase)
-					if !ok {
-						testCases = []testCase{}
-					}
-					testCases = append(testCases, testcase)
-					failureTestCasesMap[check] = testCases
-				}
 			}
 		}
 	}

test/webhook_test.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 set -e
 
-# Testing to ensure that the webhook starts up, allows a correct deployment to pass,
+echo "Testing to ensure that the webhook starts up, allows a correct deployment to pass, and prevents a incorrectly formatted deployment."
-# and prevents a incorrectly formatted deployment.
+
 BLUE='\033[0;34m'
 GREEN='\033[0;32m'
 RED='\033[0;31m'
@@ -57,9 +57,11 @@ function clean_up() {
         # || true to avoid issues when we cannot delete
         kubectl delete -f $filename ||true
     done
-    # Uninstall webhook and webhook config
+    echo "Uninstalling webhook and webhook config"
-    kubectl delete validatingwebhookconfigurations polaris-webhook --wait=false
+    kubectl delete validatingwebhookconfigurations polaris-webhook --wait=false || true
-    kubectl -n polaris delete deploy -l app=polaris --wait=false
+    kubectl delete validatingwebhookconfigurations polaris-validate-webhook --wait=false || true
+    kubectl delete validatingwebhookconfigurations polaris-mutate-webhook --wait=false || true
+    kubectl -n polaris delete deploy -l app=polaris --wait=false || true
     echo -e "\n\nDone cleaning up\n\n"
 }
 
@@ -77,26 +79,26 @@ kubectl create ns scale-test
 kubectl create ns polaris
 kubectl create ns tests
 
-# Install a bad deployment
+echo "Installing a bad deployment"
 kubectl apply -n scale-test -f ./test/webhook_cases/failing_test.deployment.yaml
 
-# Install the webhook
+echo "Installing the webhook"
 helm repo add fairwinds-stable https://charts.fairwinds.com/stable
 helm install polaris fairwinds-stable/polaris --namespace polaris --create-namespace \
   --set dashboard.enable=false \
   --set webhook.enable=true \
   --set image.tag=$CI_SHA1
 
-# wait for the webhook to come online
+echo "Waiting for the webhook to come online"
 check_webhook_is_ready
 sleep 5
 
 kubectl logs -n polaris $(kubectl get po -oname -n polaris | grep webhook) --follow &
 
-# Webhook started, setting all tests as passed initially.
+echo "Webhook started"
 ALL_TESTS_PASSED=1
 
-# Run tests against correctly configured objects
+echo "Running tests against correctly configured objects"
 for filename in test/webhook_cases/passing_test.*.yaml; do
     echo -e "\n\n"
     echo -e "${BLUE}TEST CASE: $filename${NC}"
@@ -109,7 +111,7 @@ for filename in test/webhook_cases/passing_test.*.yaml; do
     kubectl delete -n tests -f $filename || true
 done
 
-# Run tests against incorrectly configured objects
+echo "Running tests against incorrectly configured objects"
 for filename in test/webhook_cases/failing_test.*.yaml; do
     echo -e "\n\n"
     echo -e "${BLUE}TEST CASE: $filename${NC}"
@@ -123,6 +125,7 @@ for filename in test/webhook_cases/failing_test.*.yaml; do
     kubectl delete -n tests -f $filename || true
 done
 
+echo "Checking ability to scale"
 kubectl -n scale-test scale deployment nginx-deployment --replicas=2
 sleep 5
 kubectl get po -n scale-test
@@ -133,9 +136,12 @@ if [ $pod_count != 2 ]; then
 fi
 
 if [ -z $SKIP_FINAL_CLEANUP ]; then
+  echo "Doing final cleanup..."
   clean_up
 fi
 
+echo "Done with tests"
+
 #Verify that all the tests passed.
 if [ $ALL_TESTS_PASSED -eq 1 ]; then
     echo "Tests Passed."