Merge pull request #24 from ttousai/issue-19

Update controls to CIS Kubernetes Benchmark v1.1.0

.travis.yml

@@ -2,4 +2,5 @@ language: go
 install: 
   - go get github.com/aquasecurity/kube-bench
 script:
+  - go test ./...
   - go build -o kube-bench .

README.md

@@ -30,18 +30,25 @@ If Go is installed on the target machines, you can simply clone this repository
 
 ```
 Available Commands:
-  master      Checks for Kubernetes master node
-  node        Checks for Kubernetes node
-  federated   Checks for Kubernetes federated deployment
-  help        Help information
+  federated   Run benchmark checks for a Kubernetes federated deployment.
+  help        Help about any command
+  master      Run benchmark checks for a Kubernetes master node.
+  node        Run benchmark checks for a Kubernetes node.
 
 Flags:
-  -c, --check string   A comma-delimited list of checks to run as specified in CIS document. Example --check="1.1.1,1.1.2"
-  -g, --group string   Run all the checks under this comma-delimited list of groups. Example --group="1.1"
-  -h, --help           help for kube-bench
-  --json               Output results as JSON
+  -c, --check string          A comma-delimited list of checks to run as specified in CIS document. Example --check="1.1.1,1.1.2"
+      --config string         config file (default is ./cfg/config.yaml)
+  -g, --group string          Run all the checks under this comma-delimited list of groups. Example --group="1.1"
+      --installation string   Specify how kubernetes cluster was installed. Possible values are default,hyperkube,kops,kubeadm (default "default")
+      --json                  Prints the results as JSON
+  -v, --verbose               verbose output (default false)
 ```
 
+## Configuration
+Kubernetes config and binary file locations and names can vary from installation to installation, so these are configurable in the `cfg/config.yaml` file. 
+
+They also tend to vary according to which tool was used to install Kubernetes. You can use the `--installation` flag to pick up a different default set of file names and locations. Again these defaults are configurable through `cfg/config.yaml` (and pull requests to correct or add default file locations are especially welcome). 
+
 ## Test config YAML representation
 The tests are represented as YAML documents (installed by default into ./cfg).
 

cfg/config.yaml

@@ -7,11 +7,106 @@
 # nodeControls: ./cfg/node.yaml
 # federatedControls: ./cfg/federated.yaml
 
-## Configuration Directories.
-# Specifies the directories to look for configuration files
-# for the kubernetes components.
-#
-## Uncomment to use different paths.
-# kubeConfDir: /etc/kubernetes
-# etcdConfDir: /etc/etcd
-# flanneldConfDir: /etc/sysconfig
+## Support components
+etcd:
+  bin: etcd
+  conf: /etc/etcd/etcd.conf
+
+flanneld:
+  bin: flanneld
+  conf: /etc/sysconfig/flanneld
+
+# Installation
+# Configure kubernetes component binaries and paths to their configuration files.
+installation:
+  default:
+    config: /etc/kubernetes/config
+    master:
+      bin:
+        apiserver: apiserver
+        scheduler: scheduler
+        controller-manager: controller-manager
+      conf:
+        apiserver: /etc/kubernetes/apiserver
+        scheduler: /etc/kubernetes/scheduler
+        controller-manager: /etc/kubernetes/apiserver
+    node:
+      bin:
+        kubelet: kubelet
+        proxy: proxy
+      conf:
+        kubelet: /etc/kubernetes/kubelet
+        proxy: /etc/kubernetes/proxy
+    federated:
+      bin:
+        apiserver: federation-apiserver
+        controller-manager: federation-controller-manager
+
+  kops:
+    config: /etc/kubernetes/config
+    master:
+      bin:
+        apiserver: apiserver
+        scheduler: scheduler
+        controller-manager: controller-manager
+      conf:
+        apiserver: /etc/kubernetes/apiserver
+        scheduler: /etc/kubernetes/scheduler
+        controller-manager: /etc/kubernetes/apiserver
+    node:
+      bin:
+        kubelet: kubelet
+        proxy: proxy
+      conf:
+        kubelet: /etc/kubernetes/kubelet
+        proxy: /etc/kubernetes/proxy
+    federated:
+      bin:
+        apiserver: federation-apiserver
+        controller-manager: federation-controller-manager
+
+  hyperkube:
+    config: /etc/kubernetes/config
+    master:
+      bin:
+        apiserver: hyperkube apiserver
+        scheduler: hyperkube scheduler
+        controller-manager: hyperkube controller-manager
+      conf:
+        apiserver: /etc/kubernetes/apiserver
+        scheduler: /etc/kubernetes/scheduler
+        controller-manager: /etc/kubernetes/apiserver
+    node:
+      bin:
+        kubelet: hyperkube kubelet
+        proxy: hyperkube proxy
+      conf:
+        kubelet: /etc/kubernetes/kubelet
+        proxy: /etc/kubernetes/proxy
+    federated:
+      bin:
+        apiserver: hyperkube federation-apiserver
+        controller-manager: hyperkube federation-controller-manager
+
+  kubeadm:
+    config: /etc/kubernetes/config
+    master:
+      bin:
+        apiserver: kube-apiserver
+        scheduler: kube-scheduler
+        controller-manager: kube-controller-manager
+      conf:
+        apiserver: /etc/kubernetes/admin.conf
+        scheduler: /etc/kubernetes/scheduler.conf
+        controller-manager: /etc/kubernetes/controller-manager.conf
+    node:
+      bin:
+        kubelet: kubelet
+        proxy: kube-proxy
+      conf:
+        kubelet: /etc/kubernetes/kubelet.conf
+        proxy: /etc/kubernetes/proxy.conf
+    federated:
+      bin:
+        apiserver: kube-federation-apiserver
+        controller-manager: kube-federation-controller-manager

cfg/federated.yaml

@@ -9,7 +9,7 @@ groups:
   checks:
      - id: 3.1.1
        text: "Ensure that the --anonymous-auth argument is set to false (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
            - flag: "--anonymous-auth"
@@ -23,7 +23,7 @@ groups:
 
      - id: 3.1.2
        text: "Ensure that the --basic-auth-file argument is not set (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
            - flag: "--basic-auth-file"
@@ -35,7 +35,7 @@ groups:
   
      - id: 3.1.3
        text: "Ensure that the --insecure-allow-any-token argument is not set (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
            - flag: "--insecure-allow-any-token"
@@ -46,7 +46,7 @@ groups:
  
      - id: 3.1.4
        text: "Ensure that the --insecure-bind-address argument is not set (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
            - flag: "--insecure-bind-address"
@@ -57,7 +57,7 @@ groups:
 
      - id: 3.1.5
        text: "Ensure that the --insecure-port argument is set to 0 (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--insecure-port"
@@ -71,7 +71,7 @@ groups:
 
      - id: 3.1.6
        text: "Ensure that the --secure-port argument is not set to 0 (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests: 
          bin_op: or
          test_items:
@@ -88,7 +88,7 @@ groups:
 
      - id: 3.1.7
        text: "Ensure that the --profiling argument is set to false (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--profiling"
@@ -102,7 +102,7 @@ groups:
 
      - id: 3.1.8
        text: "Ensure that the admission control policy is not set to AlwaysAdmit (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--admission-control"
@@ -117,7 +117,7 @@ groups:
 
      - id: 3.1.9
        text: "Ensure that the admission control policy is set to NamespaceLifecycle (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "admission-control"
@@ -131,7 +131,7 @@ groups:
 
      - id: 3.1.10
        text: "Ensure that the --audit-log-path argument is set as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--audit-log-path"
@@ -142,7 +142,7 @@ groups:
   
      - id: 3.1.11
        text: "Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--audit-log-maxage"
@@ -156,7 +156,7 @@ groups:
                
      - id: 3.1.12
        text: "Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--audit-log-maxbackup"
@@ -170,7 +170,7 @@ groups:
 
      - id: 3.1.13
        text: "Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--audit-log-maxsize"
@@ -184,7 +184,7 @@ groups:
 
      - id: 3.1.14
        text: "Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--authorization-mode"
@@ -198,7 +198,7 @@ groups:
  
      - id: 3.1.15
        text: "Ensure that the --token-auth-file parameter is not set (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--token-auth-file"
@@ -210,7 +210,7 @@ groups:
   
      - id: 3.1.16
        text: "Ensure that the --service-account-lookup argument is set to true (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--service-account-lookup"
@@ -224,7 +224,7 @@ groups:
 
      - id: 3.1.17
        text: "Ensure that the --service-account-key-file argument is set as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--service-account-key-file"
@@ -235,7 +235,7 @@ groups:
 
      - id: 3.1.18
        text: "Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Scored"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          bin_op: and
          test_items:
@@ -252,7 +252,7 @@ groups:
 
      - id: 3.1.19
        text: "Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          bin_op: and
          test_items:
@@ -271,7 +271,7 @@ groups:
   checks:
      - id: 3.2.1
        text: "Ensure that the --profiling argument is set to false (Scored)"
-       audit: "ps -ef | grep federation-controller-manager | grep -v grep"
+       audit: "ps -ef | grep $fedcontrollermanagerbin | grep -v grep"
        tests:
          test_items:
          - flag: "--profiling"

cfg/master.yaml

@@ -9,7 +9,7 @@ groups:
   checks:
     - id: 1.1.1
       text: "Ensure that the --allow-privileged argument is set to false (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "allow-privileged"
@@ -17,13 +17,13 @@ groups:
             op: eq
             value: false
           set: true
-      remediation: "Edit the $kubeConfDir/config file on the master node and set 
+      remediation: "Edit the $apiserverconf file on the master node and set 
               the KUBE_ALLOW_PRIV parameter to \"--allow-privileged=false\""
       scored: true
 
     - id: 1.1.2
       text: "Ensure that the --anonymous-auth argument is set to false (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--anonymous-auth"
@@ -31,37 +31,37 @@ groups:
             op: eq
             value: false
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set 
+      remediation: "Edit the $apiserverconf file on the master node and set 
               the KUBE_API_ARGS parameter to \"--anonymous-auth=false\""
       scored: true
 
     - id: 1.1.3
       text: "Ensure that the --basic-auth-file argument is not set (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--basic-auth-file"
           set: false
       remediation: "Follow the documentation and configure alternate mechanisms for 
-              authentication. Then, edit the $kubeConfDir/apiserver file on the master 
+              authentication. Then, edit the $apiserverconf file on the master 
               node and remove the \"--basic-auth-file=<filename>\" argument from the 
               KUBE_API_ARGS parameter."
       scored: true
 
     - id: 1.1.4
       text: "Ensure that the --insecure-allow-any-token argument is not set (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag:  "--insecure-allow-any-token"
           set: false
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and remove 
+      remediation: "Edit the $apiserverconf file on the master node and remove 
               the --insecure-allow-any-token argument from the KUBE_API_ARGS parameter."
       scored: true
 
     - id: 1.1.5
       text: "Ensure that the --kubelet-https argument is set to true (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests: 
         bin_op: or
         test_items:
@@ -72,24 +72,24 @@ groups:
           set: true
         - flag: "--kubelet-https"
           set: false
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and remove 
+      remediation: "Edit the $apiserverconf file on the master node and remove 
               the --kubelet-https argument from the KUBE_API_ARGS parameter."
       scored: true
 
     - id: 1.1.6
       text: "Ensure that the --insecure-bind-address argument is not set (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--insecure-bind-address"
           set: false
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and remove 
+      remediation: "Edit the $apiserverconf file on the master node and remove 
               the --insecure-bind-address argument from the KUBE_API_ADDRESS parameter."
       scored: true
 
     - id: 1.1.7
       text: "Ensure that the --insecure-port argument is set to 0 (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--insecure-port"
@@ -97,13 +97,13 @@ groups:
             op: eq
             value: 0
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set
+      remediation: "Edit the $apiserverconf file on the master node and set
               --insecure-port=0 in the KUBE_API_PORT parameter."
       scored: true
 
     - id: 1.1.8
       text: "Ensure that the --secure-port argument is not set to 0 (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests: 
         bin_op: or
         test_items:
@@ -114,14 +114,14 @@ groups:
             set: true
           - flag: "--secure-port"
             set: false
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and either 
+      remediation: "Edit the $apiserverconf file on the master node and either 
               remove the --secure-port argument from the KUBE_API_ARGS parameter or set 
               it to a different desired port."
       scored: true
 
     - id: 1.1.9
       text: "Ensure that the --profiling argument is set to false (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--profiling"
@@ -129,13 +129,13 @@ groups:
             op: eq
             value: false
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to \"--profiling=false\""
       scored: true
 
     - id: 1.1.10
       text: "Ensure that the --repair-malformed-updates argument is set to false (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--repair-malformed-updates"
@@ -143,13 +143,13 @@ groups:
             op: eq
             value: false
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to \"--repair-malformed-updates=false\""
       scored: true
 
     - id: 1.1.11
       text: "Ensure that the admission control policy is not set to AlwaysAdmit (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--admission-control"
@@ -157,13 +157,13 @@ groups:
             op: nothave
             value: AlwaysAdmit 
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_ADMISSION_CONTROL parameter to a value that does not include AlwaysAdmit"
       scored: true
  
     - id: 1.1.12
       text: "Ensure that the admission control policy is set to AlwaysPullImages (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--admission-control"
@@ -171,13 +171,13 @@ groups:
             op: has
             value: "AlwaysPullImages"
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_ADMISSION_CONTROL parameter to \"--admission-control=...,AlwaysPullImages,...\""
       scored: true
  
     - id: 1.1.13
       text: "Ensure that the admission control policy is set to DenyEscalatingExec (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--admission-control"
@@ -185,13 +185,13 @@ groups:
             op: has
             value: "DenyEscalatingExec"
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_ADMISSION_CONTROL parameter to \"--admission-control=...,DenyEscalatingExec,...\""
       scored: true
 
     - id: 1.1.14
       text: "Ensure that the admission control policy is set to SecurityContextDeny (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--admission-control"
@@ -199,13 +199,13 @@ groups:
             op: has
             value: "SecurityContextDeny"
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_ADMISSION_CONTROL parameter to \"--admission-control=...,SecurityContextDeny,...\""
       scored: true
 
     - id: 1.1.15
       text: "Ensure that the admission control policy is set to NamespaceLifecycle (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "admission-control"
@@ -213,24 +213,24 @@ groups:
             op: has
             value: "NamespaceLifecycle"
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_ADMISSION_CONTROL parameter to \"--admission-control=NamespaceLifecycle,...\""
       scored: true
 
     - id: 1.1.16
       text: "Ensure that the --audit-log-path argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--audit-log-path"
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to \"--audit-log-path=<filename>\""
       scored: true
 
     - id: 1.1.17
       text: "Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--audit-log-maxage"
@@ -238,13 +238,13 @@ groups:
             op: gte
             value: 30
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to \"--audit-log-maxage=30\""
       scored: true
 
     - id: 1.1.18
       text: "Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--audit-log-maxbackup"
@@ -252,13 +252,13 @@ groups:
             op: gte
             value: 10
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to \"--audit-log-maxbackup=10\""
       scored: true
 
     - id: 1.1.19
       text: "Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--audit-log-maxsize"
@@ -266,13 +266,13 @@ groups:
             op: gte
             value: 100
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to \"--audit-log-maxsize=100\""
       scored: true
 
     - id: 1.1.20
       text: "Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--authorization-mode"
@@ -280,38 +280,38 @@ groups:
             op: nothave
             value: "AlwaysAllow"
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the 
+      remediation: "Edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to values other than \"--authorization-mode=AlwaysAllow\""
       scored: true
 
     - id: 1.1.21
       text: "Ensure that the --token-auth-file parameter is not set (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--token-auth-file"
           set: false
       remediation: "Follow the documentation and configure alternate mechanisms for authentication. 
-              Then, edit the $kubeConfDir/apiserver file on the master node and remove the 
+              Then, edit the $apiserverconf file on the master node and remove the 
               \"--tokenauth-file=<filename>\" argument from the KUBE_API_ARGS parameter."
       scored: true
  
     - id: 1.1.22
       text: "Ensure that the --kubelet-certificate-authority argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--kubelet-certificate-authority"
           set: true
       remediation: "Follow the Kubernetes documentation and setup the TLS connection between 
-              the apiserver and kubelets. Then, edit the $kubeConfDir/apiserver file on the 
+              the apiserver and kubelets. Then, edit the $apiserverconf file on the 
               master node and set the KUBE_API_ARGS parameter to 
               \"--kubelet-certificate-authority=<ca-string>\""
       scored: true
 
     - id: 1.1.23
       text: "Ensure that the --kubelet-client-certificate and --kubelet-clientkey arguments are set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         bin_op: and
         test_items:
@@ -320,14 +320,14 @@ groups:
         - flag: "--kubelet-client-key"
           set: true
       remediation: "Follow the Kubernetes documentation and set up the TLS connection between the apiserver 
-              and kubelets. Then, edit the $kubeConfDir/apiserver file on the master node and set the 
+              and kubelets. Then, edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to \"--kubelet-clientcertificate=<path/to/client-certificate-file>\" 
               and \"--kubelet-clientkey=<path/to/client-key-file>\""
       scored: true
 
     - id: 1.1.24
       text: "Ensure that the --service-account-lookup argument is set to true (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--service-account-lookup"
@@ -335,13 +335,13 @@ groups:
             op: eq
             value: true
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the KUBE_API_ARGS parameter 
+      remediation: "Edit the $apiserverconf file on the master node and set the KUBE_API_ARGS parameter 
               to \"--service-account-lookup=true\""
       scored: true
  
     - id: 1.1.25
       text: "Ensure that the admission control policy is set to PodSecurityPolicy (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--admission-control"
@@ -350,24 +350,24 @@ groups:
             value: "PodSecurityPolicy"
           set: true
       remediation: "Follow the documentation and create Pod Security Policy objects as per your environment. 
-              Then, edit the $kubeConfDir/apiserver file on the master node and set the KUBE_ADMISSION_CONTROL 
+              Then, edit the $apiserverconf file on the master node and set the KUBE_ADMISSION_CONTROL 
               parameter to \"--admission-control=...,PodSecurityPolicy,...\""
       scored: true
 
     - id: 1.1.26
       text: "Ensure that the --service-account-key-file argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--service-account-key-file"
           set: true
-      remediation: "Edit the $kubeConfDir/apiserver file on the master node and set the KUBE_API_ARGS 
+      remediation: "Edit the $apiserverconf file on the master node and set the KUBE_API_ARGS 
               parameter to \"--service-account-key-file=<filename>\""
       scored: true
 
     - id: 1.1.27
       text: "Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Scored"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         bin_op: and
         test_items:
@@ -376,14 +376,14 @@ groups:
         - flag: "--etcd-keyfile"
           set: true
       remediation: "Follow the Kubernetes documentation and set up the TLS connection between the apiserver 
-              and etcd. Then, edit the $kubeConfDir/apiserver file on the master node and set the 
+              and etcd. Then, edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to include \"--etcd-certfile=<path/to/clientcertificate-file>\" 
               and \"--etcd-keyfile=<path/to/client-key-file>\""
       scored: true
  
     - id: 1.1.28
       text: "Ensure that the admission control policy is set to ServiceAccount (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--admission-control"
@@ -392,13 +392,13 @@ groups:
             value: "ServiceAccount"
           set: true
       remediation: "Follow the documentation and create ServiceAccount objects as per your environment. 
-              Then, edit the $kubeConfDir/apiserver file on the master node and set the 
+              Then, edit the $apiserverconf file on the master node and set the 
               KUBE_ADMISSION_CONTROL parameter to \"--admissioncontrol=...,ServiceAccount,...\""
       scored: true
 
     - id: 1.1.29
       text: "Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         bin_op: and
         test_items:
@@ -407,41 +407,97 @@ groups:
         - flag: "--tls-private-key-file"
           set: true
       remediation: "Follow the Kubernetes documentation and set up the TLS connection on the apiserver. 
-              Then, edit the $kubeConfDir/apiserver file on the master node and set the KUBE_API_ARGS parameter to 
+              Then, edit the $apiserverconf file on the master node and set the KUBE_API_ARGS parameter to 
               include \"--tls-cert-file=<path/to/tls-certificatefile>\" and 
               \"--tls-private-key-file=<path/to/tls-key-file>\""
       scored: true
 
     - id: 1.1.30
       text: "Ensure that the --client-ca-file argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--client-ca-file"
           set: true
       remediation: "Follow the Kubernetes documentation and set up the TLS connection on the apiserver. 
-              Then, edit the $kubeConfDir/apiserver file on the master node and set the 
+              Then, edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to include \"--client-ca-file=<path/to/client-ca-file>\""
       scored: true
 
     - id: 1.1.31
       text: "Ensure that the --etcd-cafile argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
       tests:
         test_items:
         - flag: "--etcd-cafile"
           set: true
       remediation: "Follow the Kubernetes documentation and set up the TLS connection between the apiserver 
-              and etcd. Then, edit the $kubeConfDir/apiserver file on the master node and set the 
+              and etcd. Then, edit the $apiserverconf file on the master node and set the 
               KUBE_API_ARGS parameter to include \"--etcd-cafile=<path/to/ca-file>\""
       scored: true
 
+    - id: 1.1.32
+      text: "Ensure that the --authorization-mode argument is set to Node (Scored)"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
+      tests:
+        test_items:
+        - flag: "--authorization-mode"
+          compare:
+            op: has
+            value: "Node"
+          set: true
+      remediation: "Edit the /etc/kubernetes/apiserver file on the master node and set the KUBE_API_ARGS
+              parameter to a value to include --authorization-mode=Node. One such example could be
+              as below:\n
+              KUBE_API_ARGS=\"--authorization-mode=Node,RBAC\""
+      scored: true
+
+    - id: 1.1.33
+      text: "Ensure that the admission control policy is set to NodeRestriction (Scored)"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
+      tests:
+        test_items:
+        - flag: "--admission-control"
+          compare:
+            op: has
+            value: "NodeRestriction"
+          set: true
+      remediation: "Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets.
+              Then, edit the /etc/kubernetes/apiserver file on the master node and set the
+              KUBE_ADMISSION_CONTROL parameter to \"--admissioncontrol=...,NodeRestriction,...\""
+      scored: true
+
+    - id: 1.1.34
+      text: "1.1.34 Ensure that the --experimental-encryption-provider-config argument is set as appropriate (Scored)"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
+      tests:
+        test_items:
+        - flag: "--experimental-encryption-provider-config"
+          set: true
+      remediation: "Follow the Kubernetes documentation and configure a EncryptionConfig file. Then, edit
+              the /etc/kubernetes/apiserver file on the master node and set the KUBE_API_ARGS
+              parameter to \"--experimental-encryption-provider-config=</path/to/EncryptionConfig/File>\""
+      scored: true
+
+# TODO: provide flag to WARN of manual tasks which we can't automate.
+    - id: 1.1.35
+      text: "Ensure that the encryption provider is set to aescbc (Scored)"
+      audit: "ps -ef | grep $apiserverbin | grep -v grep"
+      tests:
+        test_items:
+        - flag: "requires manual intervention"
+          set: true
+      remediation: "Follow the Kubernetes documentation and configure a EncryptionConfig file. In this file,
+              choose aescbc as the encryption provider"
+      scored: true
+
+
 - id: 1.2
   text: "Scheduler"
   checks:
     - id: 1.2.1
       text: "Ensure that the --profiling argument is set to false (Scored)"
-      audit: "ps -ef | grep kube-scheduler | grep -v grep"
+      audit: "ps -ef | grep $schedulerbin | grep -v grep"
       tests:
         test_items:
         - flag: "--profiling"
@@ -449,7 +505,7 @@ groups:
             op: eq
             value: false
           set: true
-      remediation: "Edit the $kubeConfDir/scheduler file on the master node and set the KUBE_SCHEDULER_ARGS 
+      remediation: "Edit the $schedulerconf file on the master node and set the KUBE_SCHEDULER_ARGS 
               parameter to \"--profiling=false\""
       scored: true
 
@@ -458,18 +514,18 @@ groups:
   checks:
     - id: 1.3.1
       text: "Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-controller-manager | grep -v grep"
+      audit: "ps -ef | grep $controllermanagerbin | grep -v grep"
       tests:
         test_items:
           - flag: "--terminated-pod-gc-threshold"
             set: true
-      remediation: "Edit the $kubeConfDir/controller-manager file on the master node and set the 
+      remediation: "Edit the $controllermanagerconf file on the master node and set the 
               KUBE_CONTROLLER_MANAGER_ARGS parameter to \"--terminated-pod-gcthreshold=<appropriate-number>\""
       scored: true
  
     - id: 1.3.2
       text: "Ensure that the --profiling argument is set to false (Scored)"
-      audit: "ps -ef | grep kube-controller-manager | grep -v grep"
+      audit: "ps -ef | grep $controllermanagerbin | grep -v grep"
       tests:
         test_items:
         - flag: "--profiling"
@@ -477,25 +533,13 @@ groups:
             op: eq
             value: false
           set: true
-      remediation: "Edit the $kubeConfDir/controller-manager file on the master node and set the 
+      remediation: "Edit the $controllermanagerconf file on the master node and set the 
               KUBE_CONTROLLER_MANAGER_ARGS parameter to \"--profiling=false\""
       scored: true
  
     - id: 1.3.3
-      text: "Ensure that the --insecure-experimental-approve-all-kubelet-csrs-for-group argument is not set (Scored)"
-      audit: "ps -ef | grep kube-controller-manager | grep -v grep"
-      tests:
-        test_items:
-        - flag: "--insecure-experimental-approve-all-kubelet-csrs-for-group"
-          set: false
-      remediation: "Edit the $kubeConfDir/controller-manager file on the master node and remove 
-              the -insecure-experimental-approve-all-kubelet-csrs-for-group argument from the 
-              KUBE_CONTROLLER_MANAGER_ARGS parameter"
-      scored: true
-
-    - id: 1.3.4
       text: "Ensure that the --use-service-account-credentials argument is set"
-      audit: "ps -ef | grep kube-controller-manager | grep -v grep"
+      audit: "ps -ef | grep $controllermanagerbin | grep -v grep"
       tests:
         test_items:
         - flag: "--use-service-account-credentials"
@@ -503,157 +547,172 @@ groups:
             op: eq
             value: true
           set: true
-      remediation: "Edit the $kubeConfDir/controller-manager file on the master node and set the 
+      remediation: "Edit the $controllermanagerconf file on the master node and set the 
               KUBE_CONTROLLER_MANAGER_ARGS parameter to --use-service-account-credentials=true"
       scored: true
 
-    - id: 1.3.5
+    - id: 1.3.4
       text: "Ensure that the --service-account-private-key-file argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-controller-manager | grep -v grep"
+      audit: "ps -ef | grep $controllermanagerbin | grep -v grep"
       tests:
         test_items:
         - flag: "--service-account-private-key-file"
           set: true
-      remediation: "Edit the $kubeConfDir/controller-manager file on the master node and set the 
+      remediation: "Edit the $controllermanagerconf file on the master node and set the 
               KUBE_CONTROLLER_MANAGER_ARGS parameter to --service-account-private-keyfile=<filename>"
       scored: true
 
-    - id: 1.3.6
+    - id: 1.3.5
       text: "Ensure that the --root-ca-file argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-controller-manager | grep -v grep"
+      audit: "ps -ef | grep $controllermanagerbin | grep -v grep"
       tests:
         test_items:
         - flag: "--root-ca-file"
           set: true
-      remediation: "Edit the $kubeConfDir/controller-manager file on the master node and set the 
+      remediation: "Edit the $controllermanagerconf file on the master node and set the 
               KUBE_CONTROLLER_MANAGER_ARGS parameter to include --root-ca-file=<file>"
       scored: true
+
+# TODO: 1.3.6 is manual, provide way to WARN
+
+    - id: 1.3.7
+      text: " Ensure that the RotateKubeletServerCertificate argument is set to true (Scored)"
+      audit: "ps -ef | grep $controllermanagerbin | grep -v grep"
+      tests:
+        test_items:
+        - flag: "RotateKubeletServerCertificate"
+          compare:
+            op: eq
+            value: true
+          set: true
+      remediation: "Edit the /etc/kubernetes/controller-manager file on the master node and set the
+              KUBE_CONTROLLER_MANAGER_ARGS parameter to a value to include 
+              \"--feature-gates=RotateKubeletServerCertificate=true\""
+      scored: true
  
 - id: 1.4
   text: "Configure Files"
   checks:
     - id: 1.4.1
       text: "Ensure that the apiserver file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $kubeConfDir/apiserver; then stat -c %a $kubeConfDir/apiserver; fi"
+      audit: "if test -e $apiserverconf; then stat -c %a $apiserverconf; fi"
       tests:
         test_items:
         - flag: "644"
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chmod 644 $kubeConfDir/apiserver"
+              \nFor example, chmod 644 $apiserverconf"
       scored: true
 
     - id: 1.4.2
       text: "Ensure that the apiserver file ownership is set to root:root (Scored)"
-      audit: "if test -e $kubeConfDir/apiserver; then stat -c %U:%G $kubeConfDir/apiserver; fi"
+      audit: "if test -e $apiserverconf; then stat -c %U:%G $apiserverconf; fi"
       tests:
         test_items:
         - flag: "root:root"
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chown root:root $kubeConfDir/apiserver"
+              \nFor example, chown root:root $apiserverconf"
       scored: true
 
     - id: 1.4.3
       text: "Ensure that the config file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $kubeConfDir/config; then stat -c %a $kubeConfDir/config; fi"
+      audit: "if test -e $config; then stat -c %a $config; fi"
       tests:
         test_items:
         - flag: "644"
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chmod 644 $kubeConfDir/config"
+              \nFor example, chmod 644 $config"
       scored: true
 
     - id: 1.4.4
       text: "Ensure that the config file ownership is set to root:root (Scored)"
-      audit: "if test -e $kubeConfDir/config; then stat -c %U:%G $kubeConfDir/config; fi"
+      audit: "if test -e $config; then stat -c %U:%G $config; fi"
       tests:
         test_items:
         - flag: "root:root"
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chown root:root $kubeConfDir/config"
+              \nFor example, chown root:root $config"
       scored: true
 
     - id: 1.4.5
       text: "Ensure that the scheduler file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $kubeConfDir/scheduler; then stat -c %a $kubeConfDir/scheduler; fi"
+      audit: "if test -e $schedulerconf; then stat -c %a $schedulerconf; fi"
       tests:
         test_items:
           - flag: "644"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chmod 644 $kubeConfDir/scheduler"
+              \nFor example, chmod 644 $schedulerconf"
       scored: true
 
     - id: 1.4.6
       text: "Ensure that the scheduler file ownership is set to root:root (Scored)"
-      audit: "if test -e $kubeConfDir/scheduler; then stat -c %U:%G $kubeConfDir/scheduler; fi"
+      audit: "if test -e $schedulerconf; then stat -c %U:%G $schedulerconf; fi"
       tests:
         test_items:
           - flag: "root:root"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chown root:root $kubeConfDir/scheduler"
+              \nFor example, chown root:root $schedulerconf"
       scored: true
 
     - id: 1.4.7
       text: "Ensure that the etcd.conf file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $etcdConfDir/etcd.conf; then stat -c %a $etcdConfDir/etcd.conf; fi"
+      audit: "if test -e $etcdconf; then stat -c %a $etcdconf; fi"
       tests:
         test_items:
           - flag: "644"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chmod 644 $etcdConfDir/etcd.conf"
+              \nFor example, chmod 644 $etcdconf"
       scored: true
 
     - id: 1.4.8
       text: "Ensure that the etcd.conf file ownership is set to root:root (Scored)"
-      audit: "if test -e $etcdConfDir/etcd.conf; then stat -c %U:%G $etcdConfDir/etcd.conf; fi"
+      audit: "if test -e $etcdconf; then stat -c %U:%G $etcdconf; fi"
       tests:
         test_items:
         - flag: "root:root"
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chown root:root $etcdConfDir/etcd.conf"
+              \nFor example, chown root:root $etcdconf"
       scored: true
 
     - id: 1.4.9
       text: "Ensure that the flanneld file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e /etc/sysconfig/flanneld; then stat -c %a /etc/sysconfig/flanneld; fi"
+      audit: "if test -e $flanneldconf; then stat -c %a $flanneldconf; fi"
       tests:
         test_items:
         - flag: "644"
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chmod 644 /etc/sysconfig/flanneld"
+              \nFor example, chmod 644 $flanneldconf"
       scored: true
 
     - id: 1.4.10
       text: "Ensure that the flanneld file ownership is set to root:root (Scored)"
-      audit: "if test -e /etc/sysconfig/flanneld; then stat -c %U:%G /etc/sysconfig/flanneld; fi"
+      audit: "if test -e $flanneldconf; then stat -c %U:%G $flanneldconf; fi"
       tests:
         test_items:
         - flag: "root:root"
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. 
-              \nFor example, chown root:root /etc/sysconfig/flanneld"
+              \nFor example, chown root:root $flanneldconf"
       scored: true
       
     - id: 1.4.11
       text: "Ensure that the etcd data directory permissions are set to 700 or more restrictive (Scored)"
-      # audit: ps -ef | grep etcd | grep -v grep | sed 's,.*--data-dir=\(.*\)\s*.*,\1,' | xargs stat -c %a
-      audit: "ps -ef | grep etcd | grep -v grep | grep -o data-dir=.* | cut -d= -f2 | xargs stat -c %a"
-      # audit: xargs stat -c %a /etc/etcd
+      audit: "ps -ef | grep $etcdbin | grep -v grep | grep -o data-dir=.* | cut -d= -f2 | xargs stat -c %a"
       tests:
         test_items:
         - flag: "700"
           set: true
       remediation: "On the etcd server node, get the etcd data directory, passed as an argument --data-dir ,
               from the below command:\n
-              ps -ef | grep etcd\n
+              ps -ef | grep $etcdbin\n
               Run the below command (based on the etcd data directory found above). For example,\n
               chmod 700 /var/lib/etcd/default.etcd"
       scored: true
@@ -663,7 +722,7 @@ groups:
   checks:
     - id: 1.5.1
       text: "Ensure that the --cert-file and --key-file arguments are set as appropriate (Scored)"
-      audit: "ps -ef | grep etcd | grep -v grep"
+      audit: "ps -ef | grep $etcdbin | grep -v grep"
       tests:
         test_items:
           - flag: "--cert-file"
@@ -675,7 +734,7 @@ groups:
 
     - id: 1.5.2
       text: "Ensure that the --client-cert-auth argument is set to true (Scored)"
-      audit: "ps -ef | grep etcd | grep -v grep"
+      audit: "ps -ef | grep $etcdbin | grep -v grep"
       tests:
         test_items:
           - flag: "--client-cert-auth"
@@ -683,7 +742,7 @@ groups:
               op: eq
               value: true
             set: true
-      remediation: "Edit the etcd envrironment file (for example, $etcdConfDir/etcd.conf) on the 
+      remediation: "Edit the etcd envrironment file (for example, $etcdconf) on the 
               etcd server node and set the ETCD_CLIENT_CERT_AUTH parameter to \"true\". 
               Edit the etcd startup file (for example, /etc/systemd/system/multiuser.target.wants/etcd.service) 
               and configure the startup parameter for --clientcert-auth and set it to \"${ETCD_CLIENT_CERT_AUTH}\""
@@ -691,7 +750,7 @@ groups:
 
     - id: 1.5.3
       text: "Ensure that the --auto-tls argument is not set to true (Scored)"
-      audit: "ps -ef | grep etcd | grep -v grep"
+      audit: "ps -ef | grep $etcdbin | grep -v grep"
       tests:
         bin_op: or
         test_items:
@@ -701,7 +760,7 @@ groups:
             compare:
               op: neq
               value: true
-      remediation: "Edit the etcd environment file (for example, $etcdConfDir/etcd.conf) on the etcd server 
+      remediation: "Edit the etcd environment file (for example, $etcdconf) on the etcd server 
               node and comment out the ETCD_AUTO_TLS parameter. Edit the etcd startup file (for example, 
               /etc/systemd/system/multiuser.target.wants/etcd.service) and remove the startup parameter 
               for --auto-tls."
@@ -709,7 +768,7 @@ groups:
 
     - id: 1.5.4
       text: "Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Scored)"
-      audit: "ps -ef | grep etcd | grep -v grep"
+      audit: "ps -ef | grep $etcdbin | grep -v grep"
       tests:
         test_items:
           - flag: "--peer-cert-file"
@@ -724,7 +783,7 @@ groups:
 
     - id: 1.5.5
       text: "Ensure that the --peer-client-cert-auth argument is set to true (Scored)"
-      audit: "ps -ef | grep etcd | grep -v grep"
+      audit: "ps -ef | grep $etcdbin | grep -v grep"
       tests:
         test_items:
           - flag: "--peer-client-cert-auth"
@@ -734,7 +793,7 @@ groups:
             set: true
       remediation: "Note: This recommendation is applicable only for etcd clusters. If you are using only 
               one etcd server in your environment then this recommendation is not applicable.
-              Edit the etcd environment file (for example, $etcdConfDir/etcd.conf) on the etcd server node 
+              Edit the etcd environment file (for example, $etcdconf) on the etcd server node 
               and set the ETCD_PEER_CLIENT_CERT_AUTH parameter to \"true\". Edit the etcd startup file 
               (for example, /etc/systemd/system/multiuser.target.wants/etcd.service) and configure the 
               startup parameter for --peer-client-cert-auth and set it to \"${ETCD_PEER_CLIENT_CERT_AUTH}\""
@@ -742,7 +801,7 @@ groups:
 
     - id: 1.5.6
       text: "Ensure that the --peer-auto-tls argument is not set to true (Scored)"
-      audit: "ps -ef | grep etcd | grep -v grep"
+      audit: "ps -ef | grep $etcdbin | grep -v grep"
       tests:
         bin_op: or
         test_items:
@@ -755,7 +814,7 @@ groups:
             set: true
       remediation: "Note: This recommendation is applicable only for etcd clusters. 
               If you are using only one etcd server in your environment then this recommendation is 
-              not applicable. Edit the etcd environment file (for example, $etcdConfDir/etcd.conf) 
+              not applicable. Edit the etcd environment file (for example, $etcdconf) 
               on the etcd server node and comment out the ETCD_PEER_AUTO_TLS parameter. 
               Edit the etcd startup file (for example, /etc/systemd/system/multiuser.target.wants/etcd.service) 
               and remove the startup parameter for --peer-auto-tls."
@@ -763,12 +822,12 @@ groups:
 
     - id: 1.5.7
       text: "Ensure that the --wal-dir argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep etcd | grep -v grep"
+      audit: "ps -ef | grep $etcdbin | grep -v grep"
       tests:
         test_items:
           - flag: "--wal-dir"
             set: true
-      remediation: "Edit the etcd environment file (for example, $etcdConfDir/etcd.conf) on the etcd server node 
+      remediation: "Edit the etcd environment file (for example, $etcdconf) on the etcd server node 
               and set the ETCD_WAL_DIR parameter as appropriate. Edit the etcd startup file (for example, 
               /etc/systemd/system/multiuser.target.wants/etcd.service) and configure the startup parameter for 
               --wal-dir and set it to \"${ETCD_WAL_DIR}\""
@@ -776,7 +835,7 @@ groups:
 
     - id: 1.5.8
       text: "Ensure that the --max-wals argument is set to 0 (Scored)"
-      audit: "ps -ef | grep etcd | grep -v grep"
+      audit: "ps -ef | grep $etcdbin | grep -v grep"
       tests:
         test_items:
           - flag: "--max-wals"
@@ -784,7 +843,7 @@ groups:
               op: eq
               value: 0
             set: true
-      remediation: "Edit the etcd environment file (for example, $etcdConfDir/etcd.conf) on the etcd server node 
+      remediation: "Edit the etcd environment file (for example, $etcdconf) on the etcd server node 
               and set the ETCD_MAX_WALS parameter to 0. Edit the etcd startup file (for example, 
               /etc/systemd/system/multiuser.target.wants/etcd.service) and configure the startup parameter 
               for --max-wals and set it to \"${ETCD_MAX_WALS}\"."
@@ -792,7 +851,7 @@ groups:
 
     - id: 1.5.9
       text: "Ensure that a unique Certificate Authority is used for etcd (Not Scored)"
-      audit: "ps -ef | grep etcd | grep -v grep"
+      audit: "ps -ef | grep $etcdbin | grep -v grep"
       tests:
         test_items:
           - flag: "--trusted-ca-file"

cfg/node.yaml

@@ -9,7 +9,7 @@ groups:
   checks:
     - id: 2.1.1
       text: "Ensure that the --allow-privileged argument is set to false (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--allow-privileged"
@@ -17,13 +17,13 @@ groups:
               op: eq
               value: false
             set: true
-      remediation: "Edit the $kubeConfDir/config file on each node and set the KUBE_ALLOW_PRIV 
+      remediation: "Edit the $config file on each node and set the KUBE_ALLOW_PRIV 
               parameter to \"--allow-privileged=false\""
       scored: true
 
     - id: 2.1.2
       text: "Ensure that the --anonymous-auth argument is set to false (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--anonymous-auth"
@@ -31,13 +31,13 @@ groups:
               op: eq
               value: false
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on the master node and set the 
+      remediation: "Edit the $kubeletconf file on the master node and set the 
               KUBELET_ARGS parameter to \"--anonymous-auth=false\""
       scored: true
  
     - id: 2.1.3
       text: "Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--authorization-mode"
@@ -45,25 +45,25 @@ groups:
               op: nothave
               value: "AlwaysAllow"
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the 
+      remediation: "Edit the $kubeletconf file on each node and set the 
               KUBELET_ARGS parameter to \"--authorization-mode=Webhook\""
       scored: true
    
     - id: 2.1.4
       text: "Ensure that the --client-ca-file argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--client-ca-file"
             set: true 
       remediation: "Follow the Kubernetes documentation and setup the TLS connection between 
-              the apiserver and kubelets. Then, edit the $kubeConfDir/kubelet file on each node 
+              the apiserver and kubelets. Then, edit the $kubeletconf file on each node 
               and set the KUBELET_ARGS parameter to \"--client-ca-file=<path/to/client-ca-file>\""
       scored: true
 
     - id: 2.1.5
       text: "Ensure that the --read-only-port argument is set to 0 (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--read-only-port"
@@ -71,13 +71,13 @@ groups:
               op: eq
               value: 0
             set: true 
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--read-only-port=0\""
       scored: true
  
     - id: 2.1.6
       text: "Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--streaming-connection-idle-timeout"
@@ -85,13 +85,13 @@ groups:
               op: gt
               value: 0
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--streaming-connection-idle-timeout=<appropriate-timeout-value>\""
       scored: true
 
     - id: 2.1.7
       text: "Ensure that the --protect-kernel-defaults argument is set to true (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--protect-kernel-defaults"
@@ -99,13 +99,13 @@ groups:
               op: eq
               value: true
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--protect-kernel-defaults=true\""
       scored: true
 
     - id: 2.1.8
       text: "Ensure that the --make-iptables-util-chains argument is set to true (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         bin_op: or
         test_items:
@@ -116,13 +116,13 @@ groups:
             set: true
           - flag: "--make-iptables-util-chains"
             set: false
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and remove the 
+      remediation: "Edit the $kubeletconf file on each node and remove the 
               --make-iptables-util-chains argument from the KUBELET_ARGS parameter."
       scored: true
 
     - id: 2.1.9
       text: "Ensure that the --keep-terminated-pod-volumes argument is set to false (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--keep-terminated-pod-volumes"
@@ -130,24 +130,24 @@ groups:
               op: eq
               value: false
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--keep-terminated-pod-volumes=false\""
       scored: true
 
     - id: 2.1.10
       text: "Ensure that the --hostname-override argument is not set (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--hostname-override"
             set: false
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_HOSTNAME 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_HOSTNAME 
               parameter to \"\""
       scored: true
 
     - id: 2.1.11
       text: "Ensure that the --event-qps argument is set to 0 (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--event-qps"
@@ -155,13 +155,13 @@ groups:
               op: eq
               value: 0
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--event-qps=0\""
       scored: true
 
     - id: 2.1.12
       text: "Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--tls-cert-file"
@@ -169,14 +169,14 @@ groups:
           - flag: "--tls-private-key-file"
             set: true
       remediation: "Follow the Kubernetes documentation and set up the TLS connection on the Kubelet. 
-              Then, edit the $kubeConfDir/kubelet file on the master node and set the KUBELET_ARGS 
+              Then, edit the $kubeletconf file on the master node and set the KUBELET_ARGS 
               parameter to include \"--tls-cert-file=<path/to/tls-certificate-file>\" and 
               \"--tls-private-key-file=<path/to/tls-key-file>\""
       scored: true
 
     - id: 2.1.13
       text: "Ensure that the --cadvisor-port argument is set to 0 (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--cadvisor-port"
@@ -184,75 +184,128 @@ groups:
               op: eq
               value: 0
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS parameter 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS parameter 
               to \"--cadvisor-port=0\""
       scored: true
 
+    - id: 2.1.14
+      text: "Ensure that the RotateKubeletClientCertificate argument is set to true"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
+      tests:
+        test_items:
+          - flag: "RotateKubeletClientCertificate"
+            compare:
+              op: eq
+              value: true
+            set: true
+      remediation: "Edit the /etc/kubernetes/kubelet file on each node and set the KUBELET_ARGS parameter
+              to a value to include \"--feature-gates=RotateKubeletClientCertificate=true\"."
+      scored: true
+
+    - id: 2.1.15
+      text: "Ensure that the RotateKubeletServerCertificate argument is set to true"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
+      tests:
+        test_items:
+          - flag: "RotateKubeletServerCertificate"
+            compare:
+              op: eq
+              value: true
+            set: true
+      remediation: "Edit the /etc/kubernetes/kubelet file on each node and set the KUBELET_ARGS parameter
+              to a value to include \"--feature-gates=RotateKubeletServerCertificate=true\"."
+      scored: true
+
 - id: 2.2
   text: "Configuration Files"
   checks:
     - id: 2.2.1
       text: "Ensure that the config file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $kubeConfDir/config; then stat -c %a $kubeConfDir/config; fi"
+      audit: "if test -e $config; then stat -c %a $config; fi"
       tests:
         test_items:
           - flag: "644"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chmod 644 $kubeConfDir/config"
+              \nFor example, chmod 644 $config"
       scored: true
 
     - id: 2.2.2
       text: "Ensure that the config file ownership is set to root:root (Scored)"
-      audit: "if test -e $kubeConfDir/config; then stat -c %U:%G $kubeConfDir/config; fi"
+      audit: "if test -e $config; then stat -c %U:%G $config; fi"
       tests:
         test_items:
           - flag: "root:root"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chown root:root $kubeConfDir/config"
+              \nFor example, chown root:root $config"
       scored: true
 
     - id: 2.2.3
       text: "Ensure that the kubelet file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $kubeConfDir/kubelet; then stat -c %a $kubeConfDir/kubelet; fi"
+      audit: "if test -e $kubeletconf; then stat -c %a $kubeletconf; fi"
       tests:
         test_items:
           - flag: "644"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chmod 644 $kubeConfDir/kubelet"
+              \nFor example, chmod 644 $kubeletconf"
       scored: true
 
     - id: 2.2.4
       text: "Ensure that the kubelet file ownership is set to root:root (Scored)"
-      audit: "if test -e $kubeConfDir/kubelet; then stat -c %U:%G $kubeConfDir/kubelet; fi"
+      audit: "if test -e $kubeletconf; then stat -c %U:%G $kubeletconf; fi"
       tests:
         test_items:
           - flag: "root:root"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chown root:root $kubeConfDir/kubelet"
+              \nFor example, chown root:root $kubeletconf"
       scored: true
 
     - id: 2.2.5
       text: "Ensure that the proxy file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $kubeConfDir/proxy; then stat -c %a $kubeConfDir/proxy; fi"
+      audit: "if test -e $proxyconf; then stat -c %a $proxyconf; fi"
       tests:
         test_items:
           - flag: "644"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chmod 644 $kubeConfDir/proxy"
+              \nFor example, chmod 644 $proxyconf"
       scored: true
 
     - id: 2.2.6
       text: "Ensure that the proxy file ownership is set to root:root (Scored)"
-      audit: "if test -e $kubeConfDir/proxy; then stat -c %U:%G $kubeConfDir/proxy; fi"
+      audit: "if test -e $proxyconf; then stat -c %U:%G $proxyconf; fi"
       tests:
         test_items:
           - flag: "root:root"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chown root:root $kubeConfDir/proxy"
+              \nFor example, chown root:root $proxyconf"
+      scored: true
+
+# TODO: provide flag to WARN about manual checks.
+    - id: 2.2.7
+      text: "Ensure that the certificate authorities file permissions are set to
+              644 or more restrictive (Scored)"
+      audit: "if test -e $ca-file; then stat -c %a $ca-file; fi"
+      tests:
+        test_items:
+          - flag: "644"
+            set: true
+      remediation: "Run the following command to modify the file permissions of the --client-ca-file
+              \nchmod 644 <filename>"
+      scored: true
+
+# TODO: provide flag to WARN about manual checks.
+    - id: 2.2.8
+      text: "Ensure that the client certificate authorities file ownership is set to root:root"
+      audit: "if test -e $ca-file; then stat -c %U:%G $ca-file; fi"
+      tests:
+        test_items:
+          - flag: "notexist:notexist"
+            set: true
+      remediation: "Run the following command to modify the ownership of the --client-ca-file.
+              \nchown root:root <filename>"
       scored: true

check/check.go

@@ -15,14 +15,12 @@
 package check
 
 import (
-	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"strings"
-
-	"github.com/joncalhoun/pipe"
 )
 
 // NodeType indicates the type of node (master, node, federated).
@@ -49,6 +47,13 @@ const (
 	FEDERATED NodeType = "federated"
 )
 
+func handleError(err error, context string) (errmsg string) {
+	if err != nil {
+		errmsg = fmt.Sprintf("%s, error: %s\n", context, err)
+	}
+	return
+}
+
 // Check contains information about a recommendation in the
 // CIS Kubernetes 1.6+ document.
 type Check struct {
@@ -64,8 +69,9 @@ type Check struct {
 
 // Run executes the audit commands specified in a check and outputs
 // the results.
-func (c *Check) Run() {
-	var out string
+func (c *Check) Run(verbose bool) {
+	var out bytes.Buffer
+	var errmsgs string
 
 	// Check if command exists or exit with WARN.
 	for _, cmd := range c.Commands {
@@ -77,47 +83,76 @@ func (c *Check) Run() {
 	}
 
 	// Run commands.
-	if len(c.Commands) == 0 {
+	n := len(c.Commands)
+	if n == 0 {
 		// Likely a warning message.
 		c.State = WARN
 		return
 	}
 
-	p, err := pipe.New(c.Commands...)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "init: error creating command pipeline %s\n", err)
-		os.Exit(1)
+	// Each command runs,
+	//   cmd0 out -> cmd1 in, cmd1 out -> cmd2 in ... cmdn out -> os.stdout
+	//   cmd0 err should terminate chain
+	cs := c.Commands
+
+	// Initialize command pipeline
+	cs[n-1].Stdout = &out
+	i := 1
+
+	var err error
+	errmsgs = ""
+
+	for i < n {
+		cs[i-1].Stdout, err = cs[i].StdinPipe()
+		errmsgs += handleError(
+			err,
+			fmt.Sprintf("failed to run: %s\nfailed command: %s",
+				c.Audit,
+				cs[i].Args,
+			),
+		)
+
+		i++
 	}
 
-	pr, pw := io.Pipe()
-	p.Stdout = pw
-	defer pw.Close()
-
-	if err := p.Start(); err != nil {
-		fmt.Fprintf(os.Stderr, "start: error running audit command %s\n", err)
-		os.Exit(1)
+	// Start command pipeline
+	i = 0
+	for i < n {
+		err := cs[i].Start()
+		errmsgs += handleError(
+			err,
+			fmt.Sprintf("failed to run: %s\nfailed command: %s",
+				c.Audit,
+				cs[i].Args,
+			),
+		)
+		i++
 	}
 
-	// Read output of command chain into string for check.
-	go func() {
-		defer pr.Close()
-		scanner := bufio.NewScanner(pr)
-		for scanner.Scan() {
-			out += scanner.Text()
+	// Complete command pipeline
+	i = 0
+	for i < n {
+		err := cs[i].Wait()
+		errmsgs += handleError(
+			err,
+			fmt.Sprintf("failed to run: %s\nfailed command:%s",
+				c.Audit,
+				cs[i].Args,
+			),
+		)
+
+		if i < n-1 {
+			cs[i].Stdout.(io.Closer).Close()
 		}
 
-		if err := scanner.Err(); err != nil {
-			fmt.Fprintf(os.Stderr, "error accumulating  output %s\n", err)
-			os.Exit(1)
-		}
-	}()
-
-	if err := p.Wait(); err != nil {
-		fmt.Fprintf(os.Stderr, "wait: error running audit command %s\n", err)
-		os.Exit(1)
+		i++
 	}
 
-	res := c.Tests.execute(out)
+	if verbose && errmsgs != "" {
+		fmt.Fprintf(os.Stderr, "%s\n", errmsgs)
+	}
+
+	res := c.Tests.execute(out.String())
 	if res {
 		c.State = PASS
 	} else {

check/controls.go

@@ -17,7 +17,6 @@ package check
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 
 	yaml "gopkg.in/yaml.v2"
 )
@@ -46,19 +45,16 @@ type Summary struct {
 }
 
 // NewControls instantiates a new master Controls object.
-func NewControls(t NodeType, in []byte) *Controls {
-	var err error
+func NewControls(t NodeType, in []byte) (*Controls, error) {
 	c := new(Controls)
 
-	err = yaml.Unmarshal(in, c)
+	err := yaml.Unmarshal(in, c)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
-		os.Exit(1)
+		return nil, fmt.Errorf("failed to unmarshal YAML: %s", err)
 	}
 
 	if t != c.Type {
-		fmt.Fprintf(os.Stderr, "non-%s controls file specified\n", t)
-		os.Exit(1)
+		return nil, fmt.Errorf("non-%s controls file specified", t)
 	}
 
 	// Prepare audit commands
@@ -68,11 +64,11 @@ func NewControls(t NodeType, in []byte) *Controls {
 		}
 	}
 
-	return c
+	return c, nil
 }
 
 // RunGroup runs all checks in a group.
-func (controls *Controls) RunGroup(gids ...string) Summary {
+func (controls *Controls) RunGroup(verbose bool, gids ...string) Summary {
 	g := []*Group{}
 	controls.Summary.Pass, controls.Summary.Fail, controls.Summary.Warn = 0, 0, 0
 
@@ -86,7 +82,7 @@ func (controls *Controls) RunGroup(gids ...string) Summary {
 		for _, gid := range gids {
 			if gid == group.ID {
 				for _, check := range group.Checks {
-					check.Run()
+					check.Run(verbose)
 					summarize(controls, check)
 				}
 
@@ -100,7 +96,7 @@ func (controls *Controls) RunGroup(gids ...string) Summary {
 }
 
 // RunChecks runs the checks with the supplied IDs.
-func (controls *Controls) RunChecks(ids ...string) Summary {
+func (controls *Controls) RunChecks(verbose bool, ids ...string) Summary {
 	g := []*Group{}
 	m := make(map[string]*Group)
 	controls.Summary.Pass, controls.Summary.Fail, controls.Summary.Warn = 0, 0, 0
@@ -114,7 +110,7 @@ func (controls *Controls) RunChecks(ids ...string) Summary {
 		for _, check := range group.Checks {
 			for _, id := range ids {
 				if id == check.ID {
-					check.Run()
+					check.Run(verbose)
 					summarize(controls, check)
 
 					// Check if we have already added this checks group.

check/controls_test.go

@@ -0,0 +1,32 @@
+package check
+
+import (
+	"io/ioutil"
+	"testing"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+const cfgDir = "../cfg/"
+
+// validate that the files we're shipping are valid YAML
+func TestYamlFiles(t *testing.T) {
+	files, err := ioutil.ReadDir(cfgDir)
+	if err != nil {
+		t.Fatalf("error reading %s directory: %v", cfgDir, err)
+	}
+	for _, file := range files {
+		fileName := file.Name()
+		in, err := ioutil.ReadFile(cfgDir + fileName)
+		if err != nil {
+			t.Fatalf("error opening file %s: %v", fileName, err)
+		}
+
+		c := new(Controls)
+
+		err = yaml.Unmarshal(in, c)
+		if err != nil {
+			t.Fatalf("failed to load YAML from %s: %v", fileName, err)
+		}
+	}
+}

check/data

@@ -124,3 +124,16 @@ groups:
           set: true
       remediation: "Run the below command (based on the file location on your system) on the master node. For example, chmod 644 /etc/kubernetes/apiserver"
       scored: true
+
+    - id: 2.1.14
+      text: "Ensure that the apiserver file permissions are set to 644 or more restrictive (Scored)"
+      audit: "ps -ef | grep kubelet | grep -v grep"
+      tests:
+        test_items:
+        - flag: "KubeletClient"
+          compare:
+            op: eq
+            value: true
+          set: true
+      remediation: "Run the below command (based on the file location on your system) on the master node. For example, chmod 644 /etc/kubernetes/apiserver"
+      scored: true

check/test.go

@@ -57,7 +57,7 @@ func (t *testItem) execute(s string) (result bool) {
 		isset := match
 
 		if isset && t.Compare.Op != "" {
-			pttn := t.Flag + `=(\S*) *`
+			pttn := t.Flag + `=([^\s,]*) *`
 			flagRe := regexp.MustCompile(pttn)
 			vals := flagRe.FindStringSubmatch(s)
 

check/test_test.go

@@ -30,7 +30,10 @@ func init() {
 	if err != nil {
 		panic("Failed reading test data: " + err.Error())
 	}
-	controls = NewControls(MASTER, in)
+	controls, err = NewControls(MASTER, in)
+	if err != nil {
+		panic("Failed creating test controls: " + err.Error())
+	}
 }
 
 func TestTestExecute(t *testing.T) {
@@ -84,6 +87,16 @@ func TestTestExecute(t *testing.T) {
 			"text",
 			"644",
 		},
+		{
+			controls.Groups[0].Checks[9].Tests,
+			"flag value is comma-separated",
+			"2:35 ../kubelet --features-gates=KubeletClient=true,KubeletServer=true",
+		},
+		{
+			controls.Groups[0].Checks[9].Tests,
+			"flag value is comma-separated",
+			"2:35 ../kubelet --features-gates=KubeletServer=true,KubeletClient=true",
+		},
 	}
 
 	for _, c := range cases {

cmd/common.go

@@ -18,55 +18,67 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
-	"regexp"
 	"strings"
 
 	"github.com/aquasecurity/kube-bench/check"
-	"github.com/fatih/color"
 	"github.com/spf13/viper"
 )
 
 var (
-	kubeMasterBin  = []string{"kube-apiserver", "kube-scheduler", "kube-controller-manager"}
-	kubeMasterConf = []string{}
+	apiserverBin            string
+	apiserverConf           string
+	schedulerBin            string
+	schedulerConf           string
+	controllerManagerBin    string
+	controllerManagerConf   string
+	config                  string
+	etcdBin                 string
+	etcdConf                string
+	flanneldBin             string
+	flanneldConf            string
+	kubeletBin              string
+	kubeletConf             string
+	proxyBin                string
+	proxyConf               string
+	fedApiserverBin         string
+	fedControllerManagerBin string
 
-	kubeNodeBin  = []string{"kubelet"}
-	kubeNodeConf = []string{}
-
-	kubeFederatedBin  = []string{"federation-apiserver", "federation-controller-manager"}
-	kubeFederatedConf = []string{}
+	errmsgs string
 
 	// TODO: Consider specifying this in config file.
-	kubeVersion = "Kubernetes v1.6"
-
-	// Used for variable substitution
-	symbols = map[string]string{}
-
-	// Print colors
-	colors = map[check.State]*color.Color{
-		check.PASS: color.New(color.FgGreen),
-		check.FAIL: color.New(color.FgRed),
-		check.WARN: color.New(color.FgYellow),
-		check.INFO: color.New(color.FgBlue),
-	}
+	kubeVersion = "1.7.0"
 )
 
 func runChecks(t check.NodeType) {
 	var summary check.Summary
 	var file string
 
-	// Set up for config file check.
-	kubeMasterConf = append(kubeMasterConf, viper.Get("kubeConfDir").(string)+"/apiserver")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("kubeConfDir").(string)+"/scheduler")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("kubeConfDir").(string)+"/controller-manager")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("kubeConfDir").(string)+"/config")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("etcdConfDir").(string)+"/etcd.conf")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("flanneldConfDir").(string)+"/flanneld")
-	kubeNodeConf = append(kubeNodeConf, viper.Get("kubeConfDir").(string)+"/kubelet")
-	kubeNodeConf = append(kubeNodeConf, viper.Get("kubeConfDir").(string)+"/proxy")
+	// Master variables
+	apiserverBin = viper.GetString("installation." + installation + ".master.bin.apiserver")
+	apiserverConf = viper.GetString("installation." + installation + ".master.conf.apiserver")
+	schedulerBin = viper.GetString("installation." + installation + ".master.bin.scheduler")
+	schedulerConf = viper.GetString("installation." + installation + ".master.conf.scheduler")
+	controllerManagerBin = viper.GetString("installation." + installation + ".master.bin.controller-manager")
+	controllerManagerConf = viper.GetString("installation." + installation + ".master.conf.controler-manager")
+	config = viper.GetString("installation." + installation + ".config")
 
-	verifyNodeType(t)
+	etcdBin = viper.GetString("etcd.bin")
+	etcdConf = viper.GetString("etcd.conf")
+	flanneldBin = viper.GetString("flanneld.bin")
+	flanneldConf = viper.GetString("flanneld.conf")
+
+	// Node variables
+	kubeletBin = viper.GetString("installation." + installation + ".node.bin.kubelet")
+	kubeletConf = viper.GetString("installation." + installation + ".node.conf.kubelet")
+	proxyBin = viper.GetString("installation." + installation + ".node.bin.proxy")
+	proxyConf = viper.GetString("installation." + installation + ".node.conf.proxy")
+
+	// Federated
+	fedApiserverBin = viper.GetString("installation." + installation + ".federated.bin.apiserver")
+	fedControllerManagerBin = viper.GetString("installation." + installation + ".federated.bin.controller-manager")
+
+	// Run kubernetes installation validation checks.
+	warns := verifyNodeType(t)
 
 	switch t {
 	case check.MASTER:
@@ -79,114 +91,92 @@ func runChecks(t check.NodeType) {
 
 	in, err := ioutil.ReadFile(file)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "error opening %s controls file: %s\n", t, err)
+		fmt.Fprintf(os.Stderr, "error opening %s controls file: %v\n", t, err)
 		os.Exit(1)
 	}
 
-	// Variable substitutions. Replace all occurrences of variables in controls file.
-	s := strings.Replace(string(in), "$kubeConfDir", viper.Get("kubeConfDir").(string), -1)
-	s = strings.Replace(s, "$etcdConfDir", viper.Get("etcdConfDir").(string), -1)
-	s = strings.Replace(s, "$flanneldConfDir", viper.Get("etcdConfDir").(string), -1)
+	// Variable substitutions. Replace all occurrences of variables in controls files.
+	s := strings.Replace(string(in), "$apiserverbin", apiserverBin, -1)
+	s = strings.Replace(s, "$apiserverconf", apiserverConf, -1)
+	s = strings.Replace(s, "$schedulerbin", schedulerBin, -1)
+	s = strings.Replace(s, "$schedulerconf", schedulerConf, -1)
+	s = strings.Replace(s, "$controllermanagerbin", controllerManagerBin, -1)
+	s = strings.Replace(s, "$controllermanagerconf", controllerManagerConf, -1)
+	s = strings.Replace(s, "$controllermanagerconf", controllerManagerConf, -1)
+	s = strings.Replace(s, "$config", config, -1)
 
-	controls := check.NewControls(t, []byte(s))
+	s = strings.Replace(s, "$etcdbin", etcdBin, -1)
+	s = strings.Replace(s, "$etcdconf", etcdConf, -1)
+	s = strings.Replace(s, "$flanneldbin", flanneldBin, -1)
+	s = strings.Replace(s, "$flanneldconf", flanneldConf, -1)
+
+	s = strings.Replace(s, "$kubeletbin", kubeletBin, -1)
+	s = strings.Replace(s, "$kubeletconf", kubeletConf, -1)
+	s = strings.Replace(s, "$proxybin", proxyBin, -1)
+	s = strings.Replace(s, "$proxyconf", proxyConf, -1)
+
+	s = strings.Replace(s, "$fedapiserverbin", fedApiserverBin, -1)
+	s = strings.Replace(s, "$fedcontrollermanagerbin", fedControllerManagerBin, -1)
+
+	controls, err := check.NewControls(t, []byte(s))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error setting up %s controls: %v\n", t, err)
+		os.Exit(1)
+	}
 
 	if groupList != "" && checkList == "" {
-		// log.Println("group: set, checks: not set")
 		ids := cleanIDs(groupList)
-		summary = controls.RunGroup(ids...)
-
+		summary = controls.RunGroup(verbose, ids...)
 	} else if checkList != "" && groupList == "" {
-		// log.Println("group: not set, checks: set")
 		ids := cleanIDs(checkList)
-		summary = controls.RunChecks(ids...)
-
+		summary = controls.RunChecks(verbose, ids...)
 	} else if checkList != "" && groupList != "" {
-		// log.Println("group: set, checks: set")
 		fmt.Fprintf(os.Stderr, "group option and check option can't be used together\n")
 		os.Exit(1)
-
 	} else {
-		summary = controls.RunGroup()
+		summary = controls.RunGroup(verbose)
 	}
 
-	if jsonFmt {
+	// if we successfully ran some tests and it's json format, ignore the warnings
+	if (summary.Fail > 0 || summary.Warn > 0 || summary.Pass > 0) && jsonFmt {
 		out, err := controls.JSON()
 		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to output in JSON format: %v\n", err)
+			os.Exit(1)
 		}
 
 		fmt.Println(string(out))
 	} else {
-		prettyPrint(controls, summary)
+		prettyPrint(warns, controls, summary)
 	}
 }
 
-func cleanIDs(list string) []string {
-	list = strings.Trim(list, ",")
-	ids := strings.Split(list, ",")
-
-	for _, id := range ids {
-		id = strings.Trim(id, " ")
-	}
-
-	return ids
-}
-
-func verifyNodeType(t check.NodeType) {
-	var binPath []string
-	var confPath []string
-	var out []byte
+// verifyNodeType checks the executables and config files are as expected
+// for the specified tests (master, node or federated).
+func verifyNodeType(t check.NodeType) []string {
+	var w []string
+	// Always clear out error messages.
+	errmsgs = ""
 
 	switch t {
 	case check.MASTER:
-		binPath = kubeMasterBin
-		confPath = kubeMasterConf
+		w = append(w, verifyBin(apiserverBin, schedulerBin, controllerManagerBin)...)
+		w = append(w, verifyConf(apiserverConf, schedulerConf, controllerManagerConf)...)
+		w = append(w, verifyKubeVersion(apiserverBin)...)
 	case check.NODE:
-		binPath = kubeNodeBin
-		confPath = kubeNodeConf
+		w = append(w, verifyBin(kubeletBin, proxyBin)...)
+		w = append(w, verifyConf(kubeletConf, proxyConf)...)
+		w = append(w, verifyKubeVersion(kubeletBin)...)
 	case check.FEDERATED:
-		binPath = kubeFederatedBin
-		confPath = kubeFederatedConf
+		w = append(w, verifyBin(fedApiserverBin, fedControllerManagerBin)...)
+		w = append(w, verifyKubeVersion(fedApiserverBin)...)
 	}
 
-	// These executables might not be on the user's path.
-	// TODO! Check the version number using kubectl, which is more likely to be on the path.
-	for _, b := range binPath {
-		_, err := exec.LookPath(b)
-		if err != nil {
-			colorPrint(check.WARN, fmt.Sprintf("%s: command not found on path - version check skipped\n", b))
-			continue
-		}
-
-		// Check version
-		cmd := exec.Command(b, "--version")
-		out, _ = cmd.Output()
-		if matched, _ := regexp.MatchString(kubeVersion, string(out)); !matched {
-			colorPrint(check.FAIL,
-				fmt.Sprintf(
-					"%s unsupported version, expected %s, got %s\n",
-					b,
-					kubeVersion,
-					string(out),
-				))
-			os.Exit(1)
-		}
+	if verbose {
+		fmt.Fprintf(os.Stderr, "%s\n", errmsgs)
 	}
 
-	for _, b := range binPath {
-		// Check if running.
-		cmd := exec.Command("ps", "-ef")
-		out, _ = cmd.Output()
-		if matched, _ := regexp.MatchString(".*"+b, string(out)); !matched {
-			colorPrint(check.FAIL, fmt.Sprintf("%s is not running\n", b))
-			os.Exit(1)
-		}
-	}
-
-	for _, c := range confPath {
-		if _, err := os.Stat(c); os.IsNotExist(err) {
-			colorPrint(check.WARN, fmt.Sprintf("config file %s does not exist\n", c))
-		}
-	}
+	return w
 }
 
 // colorPrint outputs the state in a specific colour, along with a message string
@@ -195,8 +185,14 @@ func colorPrint(state check.State, s string) {
 	fmt.Printf("%s", s)
 }
 
-func prettyPrint(r *check.Controls, summary check.Summary) {
-	// Print checks and results.
+// prettyPrint outputs the results to stdout in human-readable format
+func prettyPrint(warnings []string, r *check.Controls, summary check.Summary) {
+	colorPrint(check.INFO, fmt.Sprintf("Using config file: %s\n", viper.ConfigFileUsed()))
+
+	for _, w := range warnings {
+		colorPrint(check.WARN, w)
+	}
+
 	colorPrint(check.INFO, fmt.Sprintf("%s %s\n", r.ID, r.Text))
 	for _, g := range r.Groups {
 		colorPrint(check.INFO, fmt.Sprintf("%s %s\n", g.ID, g.Text))

cmd/root.go

@@ -34,9 +34,12 @@ var (
 	nodeFile      string
 	federatedFile string
 
-	kubeConfDir    string
-	etcdConfDir    string
-	flannelConfDir string
+	kubeConfDir     string
+	etcdConfDir     string
+	flanneldConfDir string
+
+	verbose      bool
+	installation string
 )
 
 // RootCmd represents the base command when called without any subcommands
@@ -59,19 +62,28 @@ func init() {
 	cobra.OnInitialize(initConfig)
 
 	RootCmd.PersistentFlags().BoolVar(&jsonFmt, "json", false, "Prints the results as JSON")
-	RootCmd.PersistentFlags().StringVarP(&checkList,
+	RootCmd.PersistentFlags().StringVar(
+		&installation,
+		"installation",
+		"default",
+		"Specify how kubernetes cluster was installed. Possible values are default,hyperkube,kops,kubeadm",
+	)
+	RootCmd.PersistentFlags().StringVarP(
+		&checkList,
 		"check",
 		"c",
 		"",
 		`A comma-delimited list of checks to run as specified in CIS document. Example --check="1.1.1,1.1.2"`,
 	)
-	RootCmd.PersistentFlags().StringVarP(&groupList,
+	RootCmd.PersistentFlags().StringVarP(
+		&groupList,
 		"group",
 		"g",
 		"",
 		`Run all the checks under this comma-delimited list of groups. Example --group="1.1"`,
 	)
 	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is ./cfg/config.yaml)")
+	RootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose output (default false)")
 }
 
 // initConfig reads in config file and ENV variables if set.
@@ -83,23 +95,13 @@ func initConfig() {
 		viper.AddConfigPath(cfgDir)   // adding ./cfg as first search path
 	}
 
-	viper.SetEnvPrefix("CISK8S")
+	viper.SetEnvPrefix("KUBE_BENCH")
 	viper.AutomaticEnv() // read in environment variables that match
 
-	// Set defaults
-	viper.SetDefault("kubeConfDir", "/etc/kubernetes")
-	viper.SetDefault("etcdConfDir", "/etc/etcd")
-	viper.SetDefault("flanneldConfDir", "/etc/sysconfig")
-
-	viper.SetDefault("masterFile", cfgDir+"/master.yaml")
-	viper.SetDefault("nodeFile", cfgDir+"/node.yaml")
-	viper.SetDefault("federatedFile", cfgDir+"/federated.yaml")
-
 	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		colorPrint(check.INFO, fmt.Sprintf("Using config file: %s\n", viper.ConfigFileUsed()))
-	} else {
-		colorPrint(check.FAIL, fmt.Sprintf("Faied to read config file: %v\n", err))
+	if err := viper.ReadInConfig(); err != nil {
+		colorPrint(check.FAIL, fmt.Sprintf("Failed to read config file: %v\n", err))
 		os.Exit(1)
 	}
+
 }

cmd/util.go

@@ -0,0 +1,113 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/aquasecurity/kube-bench/check"
+	"github.com/fatih/color"
+)
+
+var (
+	// Print colors
+	colors = map[check.State]*color.Color{
+		check.PASS: color.New(color.FgGreen),
+		check.FAIL: color.New(color.FgRed),
+		check.WARN: color.New(color.FgYellow),
+		check.INFO: color.New(color.FgBlue),
+	}
+)
+
+func handleError(err error, context string) (errmsg string) {
+	if err != nil {
+		errmsg = fmt.Sprintf("%s, error: %s\n", context, err)
+	}
+	return
+}
+
+func cleanIDs(list string) []string {
+	list = strings.Trim(list, ",")
+	ids := strings.Split(list, ",")
+
+	for _, id := range ids {
+		id = strings.Trim(id, " ")
+	}
+
+	return ids
+}
+
+func verifyConf(confPath ...string) []string {
+	var w []string
+	for _, c := range confPath {
+		if _, err := os.Stat(c); err != nil && os.IsNotExist(err) {
+			w = append(w, fmt.Sprintf("config file %s does not exist\n", c))
+		}
+	}
+
+	return w
+}
+
+func verifyBin(binPath ...string) []string {
+	var w []string
+	var binList string
+
+	// Construct proc name for ps(1)
+	for _, b := range binPath {
+		binList += b + ","
+		_, err := exec.LookPath(b)
+		errmsgs += handleError(
+			err,
+			fmt.Sprintf("%s: command not found in path", b),
+		)
+	}
+
+	binList = strings.Trim(binList, ",")
+
+	// Run ps command
+	cmd := exec.Command("ps", "-C", binList, "-o", "cmd", "--no-headers")
+	out, err := cmd.Output()
+	errmsgs += handleError(
+		err,
+		fmt.Sprintf("failed to run: %s", cmd.Args),
+	)
+
+	// Actual verification
+	for _, b := range binPath {
+		matched := strings.Contains(string(out), b)
+
+		if !matched {
+			w = append(w, fmt.Sprintf("%s is not running\n", b))
+		}
+	}
+
+	return w
+}
+
+func verifyKubeVersion(b string) []string {
+	// These executables might not be on the user's path.
+	// TODO! Check the version number using kubectl, which is more likely to be on the path.
+	var w []string
+
+	_, err := exec.LookPath(b)
+	errmsgs += handleError(
+		err,
+		fmt.Sprintf("%s: command not found on path - version check skipped", b),
+	)
+
+	// Check version
+	cmd := exec.Command(b, "--version")
+	out, err := cmd.Output()
+	errmsgs += handleError(
+		err,
+		fmt.Sprintf("failed to run:%s", cmd.Args),
+	)
+
+	matched := strings.Contains(string(out), kubeVersion)
+	if !matched {
+		w = append(w, fmt.Sprintf("%s unsupported version\n", b))
+	}
+
+	return w
+}

main.go

@@ -14,7 +14,9 @@
 
 package main
 
-import "github.com/aquasecurity/kube-bench/cmd"
+import (
+	"github.com/aquasecurity/kube-bench/cmd"
+)
 
 func main() {
 	cmd.Execute()