Merge pull request #40 from aquasecurity/roadmap

Roadmap

.travis.yml

@@ -2,4 +2,5 @@ language: go
 install: 
   - go get github.com/aquasecurity/kube-bench
 script:
+  - go test ./...
   - go build -o kube-bench .

OWNERS

@@ -0,0 +1,3 @@
+approvers:
+ - lizrice
+ - jerbia

README.md

@@ -30,18 +30,25 @@ If Go is installed on the target machines, you can simply clone this repository
 
 ```
 Available Commands:
-  master      Checks for Kubernetes master node
-  node        Checks for Kubernetes node
-  federated   Checks for Kubernetes federated deployment
-  help        Help information
+  federated   Run benchmark checks for a Kubernetes federated deployment.
+  help        Help about any command
+  master      Run benchmark checks for a Kubernetes master node.
+  node        Run benchmark checks for a Kubernetes node.
 
 Flags:
-  -c, --check string   A comma-delimited list of checks to run as specified in CIS document. Example --check="1.1.1,1.1.2"
-  -g, --group string   Run all the checks under this comma-delimited list of groups. Example --group="1.1"
-  -h, --help           help for kube-bench
-  --json               Output results as JSON
+  -c, --check string          A comma-delimited list of checks to run as specified in CIS document. Example --check="1.1.1,1.1.2"
+      --config string         config file (default is ./cfg/config.yaml)
+  -g, --group string          Run all the checks under this comma-delimited list of groups. Example --group="1.1"
+      --installation string   Specify how kubernetes cluster was installed. Possible values are default,hyperkube,kops,kubeadm (default "default")
+      --json                  Prints the results as JSON
+  -v, --verbose               verbose output (default false)
 ```
 
+## Configuration
+Kubernetes config and binary file locations and names can vary from installation to installation, so these are configurable in the `cfg/config.yaml` file. 
+
+They also tend to vary according to which tool was used to install Kubernetes. You can use the `--installation` flag to pick up a different default set of file names and locations. Again these defaults are configurable through `cfg/config.yaml` (and pull requests to correct or add default file locations are especially welcome). 
+
 ## Test config YAML representation
 The tests are represented as YAML documents (installed by default into ./cfg).
 
@@ -98,3 +105,6 @@ These operations are:
 - `lte`: tests if the flag value is less than or equal to the compared value.
 - `has`: tests if the flag value contains the compared value.
 - `nothave`: tests if the flag value does not contain the compared value.
+
+# Roadmap 
+The tests are up-to-date with the CIS Benchmark 1.1.0, which refers to Kubernetes 1.7. Going forward we should release updates to kube-bench to reflect new releases of the Benchmark, which in turn we can anticipate being made for each new Kubernetes release. 

cfg/config.yaml

@@ -7,11 +7,106 @@
 # nodeControls: ./cfg/node.yaml
 # federatedControls: ./cfg/federated.yaml
 
-## Configuration Directories.
-# Specifies the directories to look for configuration files
-# for the kubernetes components.
-#
-## Uncomment to use different paths.
-# kubeConfDir: /etc/kubernetes
-# etcdConfDir: /etc/etcd
-# flanneldConfDir: /etc/sysconfig
+## Support components
+etcd:
+  bin: etcd
+  conf: /etc/etcd/etcd.conf
+
+flanneld:
+  bin: flanneld
+  conf: /etc/sysconfig/flanneld
+
+# Installation
+# Configure kubernetes component binaries and paths to their configuration files.
+installation:
+  default:
+    config: /etc/kubernetes/config
+    master:
+      bin:
+        apiserver: apiserver
+        scheduler: scheduler
+        controller-manager: controller-manager
+      conf:
+        apiserver: /etc/kubernetes/apiserver
+        scheduler: /etc/kubernetes/scheduler
+        controller-manager: /etc/kubernetes/controller-manager
+    node:
+      bin:
+        kubelet: kubelet
+        proxy: proxy
+      conf:
+        kubelet: /etc/kubernetes/kubelet
+        proxy: /etc/kubernetes/proxy
+    federated:
+      bin:
+        apiserver: federation-apiserver
+        controller-manager: federation-controller-manager
+
+  kops:
+    config: /etc/kubernetes/config
+    master:
+      bin:
+        apiserver: apiserver
+        scheduler: scheduler
+        controller-manager: controller-manager
+      conf:
+        apiserver: /etc/kubernetes/apiserver
+        scheduler: /etc/kubernetes/scheduler
+        controller-manager: /etc/kubernetes/apiserver
+    node:
+      bin:
+        kubelet: kubelet
+        proxy: proxy
+      conf:
+        kubelet: /etc/kubernetes/kubelet
+        proxy: /etc/kubernetes/proxy
+    federated:
+      bin:
+        apiserver: federation-apiserver
+        controller-manager: federation-controller-manager
+
+  hyperkube:
+    config: /etc/kubernetes/config
+    master:
+      bin:
+        apiserver: hyperkube apiserver
+        scheduler: hyperkube scheduler
+        controller-manager: hyperkube controller-manager
+      conf:
+        apiserver: /etc/kubernetes/manifests/kube-apiserver.yaml
+        scheduler: /etc/kubernetes/manifests/kube-scheduler.yaml
+        controller-manager: /etc/kubernetes/manifests/kube-controller-manager.yaml
+    node:
+      bin:
+        kubelet: hyperkube kubelet
+        proxy: hyperkube proxy
+      conf:
+        kubelet: /etc/kubernetes/kubelet
+        proxy: /etc/kubernetes/addons/kube-proxy-daemonset.yaml
+    federated:
+      bin:
+        apiserver: hyperkube federation-apiserver
+        controller-manager: hyperkube federation-controller-manager
+
+  kubeadm:
+    config: /etc/kubernetes/config
+    master:
+      bin:
+        apiserver: kube-apiserver
+        scheduler: kube-scheduler
+        controller-manager: kube-controller-manager
+      conf:
+        apiserver: /etc/kubernetes/admin.conf
+        scheduler: /etc/kubernetes/scheduler.conf
+        controller-manager: /etc/kubernetes/controller-manager.conf
+    node:
+      bin:
+        kubelet: kubelet
+        proxy: kube-proxy
+      conf:
+        kubelet: /etc/kubernetes/kubelet.conf
+        proxy: /etc/kubernetes/proxy.conf
+    federated:
+      bin:
+        apiserver: kube-federation-apiserver
+        controller-manager: kube-federation-controller-manager

cfg/federated.yaml

@@ -9,7 +9,7 @@ groups:
   checks:
      - id: 3.1.1
        text: "Ensure that the --anonymous-auth argument is set to false (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
            - flag: "--anonymous-auth"
@@ -23,7 +23,7 @@ groups:
 
      - id: 3.1.2
        text: "Ensure that the --basic-auth-file argument is not set (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
            - flag: "--basic-auth-file"
@@ -35,7 +35,7 @@ groups:
   
      - id: 3.1.3
        text: "Ensure that the --insecure-allow-any-token argument is not set (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
            - flag: "--insecure-allow-any-token"
@@ -46,7 +46,7 @@ groups:
  
      - id: 3.1.4
        text: "Ensure that the --insecure-bind-address argument is not set (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
            - flag: "--insecure-bind-address"
@@ -57,7 +57,7 @@ groups:
 
      - id: 3.1.5
        text: "Ensure that the --insecure-port argument is set to 0 (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--insecure-port"
@@ -71,7 +71,7 @@ groups:
 
      - id: 3.1.6
        text: "Ensure that the --secure-port argument is not set to 0 (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests: 
          bin_op: or
          test_items:
@@ -88,7 +88,7 @@ groups:
 
      - id: 3.1.7
        text: "Ensure that the --profiling argument is set to false (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--profiling"
@@ -102,7 +102,7 @@ groups:
 
      - id: 3.1.8
        text: "Ensure that the admission control policy is not set to AlwaysAdmit (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--admission-control"
@@ -117,7 +117,7 @@ groups:
 
      - id: 3.1.9
        text: "Ensure that the admission control policy is set to NamespaceLifecycle (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "admission-control"
@@ -131,7 +131,7 @@ groups:
 
      - id: 3.1.10
        text: "Ensure that the --audit-log-path argument is set as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--audit-log-path"
@@ -142,7 +142,7 @@ groups:
   
      - id: 3.1.11
        text: "Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--audit-log-maxage"
@@ -156,7 +156,7 @@ groups:
                
      - id: 3.1.12
        text: "Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--audit-log-maxbackup"
@@ -170,7 +170,7 @@ groups:
 
      - id: 3.1.13
        text: "Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--audit-log-maxsize"
@@ -184,7 +184,7 @@ groups:
 
      - id: 3.1.14
        text: "Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--authorization-mode"
@@ -198,7 +198,7 @@ groups:
  
      - id: 3.1.15
        text: "Ensure that the --token-auth-file parameter is not set (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--token-auth-file"
@@ -210,7 +210,7 @@ groups:
   
      - id: 3.1.16
        text: "Ensure that the --service-account-lookup argument is set to true (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--service-account-lookup"
@@ -224,7 +224,7 @@ groups:
 
      - id: 3.1.17
        text: "Ensure that the --service-account-key-file argument is set as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          test_items:
          - flag: "--service-account-key-file"
@@ -235,7 +235,7 @@ groups:
 
      - id: 3.1.18
        text: "Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Scored"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          bin_op: and
          test_items:
@@ -252,7 +252,7 @@ groups:
 
      - id: 3.1.19
        text: "Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored)"
-       audit: "ps -ef | grep federation-apiserver | grep -v grep"
+       audit: "ps -ef | grep $fedapiserverbin | grep -v grep"
        tests:
          bin_op: and
          test_items:
@@ -271,7 +271,7 @@ groups:
   checks:
      - id: 3.2.1
        text: "Ensure that the --profiling argument is set to false (Scored)"
-       audit: "ps -ef | grep federation-controller-manager | grep -v grep"
+       audit: "ps -ef | grep $fedcontrollermanagerbin | grep -v grep"
        tests:
          test_items:
          - flag: "--profiling"

cfg/node.yaml

@@ -9,7 +9,7 @@ groups:
   checks:
     - id: 2.1.1
       text: "Ensure that the --allow-privileged argument is set to false (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--allow-privileged"
@@ -17,13 +17,13 @@ groups:
               op: eq
               value: false
             set: true
-      remediation: "Edit the $kubeConfDir/config file on each node and set the KUBE_ALLOW_PRIV 
+      remediation: "Edit the $config file on each node and set the KUBE_ALLOW_PRIV 
               parameter to \"--allow-privileged=false\""
       scored: true
 
     - id: 2.1.2
       text: "Ensure that the --anonymous-auth argument is set to false (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--anonymous-auth"
@@ -31,13 +31,13 @@ groups:
               op: eq
               value: false
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on the master node and set the 
+      remediation: "Edit the $kubeletconf file on the master node and set the 
               KUBELET_ARGS parameter to \"--anonymous-auth=false\""
       scored: true
  
     - id: 2.1.3
       text: "Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--authorization-mode"
@@ -45,25 +45,25 @@ groups:
               op: nothave
               value: "AlwaysAllow"
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the 
+      remediation: "Edit the $kubeletconf file on each node and set the 
               KUBELET_ARGS parameter to \"--authorization-mode=Webhook\""
       scored: true
    
     - id: 2.1.4
       text: "Ensure that the --client-ca-file argument is set as appropriate (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--client-ca-file"
             set: true 
       remediation: "Follow the Kubernetes documentation and setup the TLS connection between 
-              the apiserver and kubelets. Then, edit the $kubeConfDir/kubelet file on each node 
+              the apiserver and kubelets. Then, edit the $kubeletconf file on each node 
               and set the KUBELET_ARGS parameter to \"--client-ca-file=<path/to/client-ca-file>\""
       scored: true
 
     - id: 2.1.5
       text: "Ensure that the --read-only-port argument is set to 0 (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--read-only-port"
@@ -71,13 +71,13 @@ groups:
               op: eq
               value: 0
             set: true 
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--read-only-port=0\""
       scored: true
  
     - id: 2.1.6
       text: "Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--streaming-connection-idle-timeout"
@@ -85,13 +85,13 @@ groups:
               op: gt
               value: 0
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--streaming-connection-idle-timeout=<appropriate-timeout-value>\""
       scored: true
 
     - id: 2.1.7
       text: "Ensure that the --protect-kernel-defaults argument is set to true (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--protect-kernel-defaults"
@@ -99,13 +99,13 @@ groups:
               op: eq
               value: true
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--protect-kernel-defaults=true\""
       scored: true
 
     - id: 2.1.8
       text: "Ensure that the --make-iptables-util-chains argument is set to true (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         bin_op: or
         test_items:
@@ -116,13 +116,13 @@ groups:
             set: true
           - flag: "--make-iptables-util-chains"
             set: false
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and remove the 
+      remediation: "Edit the $kubeletconf file on each node and remove the 
               --make-iptables-util-chains argument from the KUBELET_ARGS parameter."
       scored: true
 
     - id: 2.1.9
       text: "Ensure that the --keep-terminated-pod-volumes argument is set to false (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--keep-terminated-pod-volumes"
@@ -130,24 +130,24 @@ groups:
               op: eq
               value: false
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--keep-terminated-pod-volumes=false\""
       scored: true
 
     - id: 2.1.10
       text: "Ensure that the --hostname-override argument is not set (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--hostname-override"
             set: false
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_HOSTNAME 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_HOSTNAME 
               parameter to \"\""
       scored: true
 
     - id: 2.1.11
       text: "Ensure that the --event-qps argument is set to 0 (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--event-qps"
@@ -155,13 +155,13 @@ groups:
               op: eq
               value: 0
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS 
               parameter to \"--event-qps=0\""
       scored: true
 
     - id: 2.1.12
       text: "Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--tls-cert-file"
@@ -169,14 +169,14 @@ groups:
           - flag: "--tls-private-key-file"
             set: true
       remediation: "Follow the Kubernetes documentation and set up the TLS connection on the Kubelet. 
-              Then, edit the $kubeConfDir/kubelet file on the master node and set the KUBELET_ARGS 
+              Then, edit the $kubeletconf file on the master node and set the KUBELET_ARGS 
               parameter to include \"--tls-cert-file=<path/to/tls-certificate-file>\" and 
               \"--tls-private-key-file=<path/to/tls-key-file>\""
       scored: true
 
     - id: 2.1.13
       text: "Ensure that the --cadvisor-port argument is set to 0 (Scored)"
-      audit: "ps -ef | grep kubelet | grep -v grep"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
       tests:
         test_items:
           - flag: "--cadvisor-port"
@@ -184,75 +184,185 @@ groups:
               op: eq
               value: 0
             set: true
-      remediation: "Edit the $kubeConfDir/kubelet file on each node and set the KUBELET_ARGS parameter 
+      remediation: "Edit the $kubeletconf file on each node and set the KUBELET_ARGS parameter 
               to \"--cadvisor-port=0\""
       scored: true
 
+    - id: 2.1.14
+      text: "Ensure that the RotateKubeletClientCertificate argument is set to true"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
+      tests:
+        test_items:
+          - flag: "RotateKubeletClientCertificate"
+            compare:
+              op: eq
+              value: true
+            set: true
+      remediation: "Edit the /etc/kubernetes/kubelet file on each node and set the KUBELET_ARGS parameter
+              to a value to include \"--feature-gates=RotateKubeletClientCertificate=true\"."
+      scored: true
+
+    - id: 2.1.15
+      text: "Ensure that the RotateKubeletServerCertificate argument is set to true"
+      audit: "ps -ef | grep $kubeletbin | grep -v grep"
+      tests:
+        test_items:
+          - flag: "RotateKubeletServerCertificate"
+            compare:
+              op: eq
+              value: true
+            set: true
+      remediation: "Edit the /etc/kubernetes/kubelet file on each node and set the KUBELET_ARGS parameter
+              to a value to include \"--feature-gates=RotateKubeletServerCertificate=true\"."
+      scored: true
+
 - id: 2.2
   text: "Configuration Files"
   checks:
     - id: 2.2.1
       text: "Ensure that the config file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $kubeConfDir/config; then stat -c %a $kubeConfDir/config; fi"
+      audit: "/bin/sh -c 'if test -e $config; then stat -c %a $config; fi'"
       tests:
+        bin_op: or
         test_items:
           - flag: "644"
+            compare:
+              op: eq
+              value: "644"
+            set: true
+          - flag: "640"
+            compare:
+              op: eq
+              value: "640"
+            set: true
+          - flag: "600"
+            compare:
+              op: eq
+              value: "600"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chmod 644 $kubeConfDir/config"
+              \nFor example, chmod 644 $config"
       scored: true
 
     - id: 2.2.2
       text: "Ensure that the config file ownership is set to root:root (Scored)"
-      audit: "if test -e $kubeConfDir/config; then stat -c %U:%G $kubeConfDir/config; fi"
+      audit: "/bin/sh -c 'if test -e $config; then stat -c %U:%G $config; fi'"
       tests:
         test_items:
           - flag: "root:root"
+            compare:
+              op: eq
+              value: root:root
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chown root:root $kubeConfDir/config"
+              \nFor example, chown root:root $config"
       scored: true
 
     - id: 2.2.3
       text: "Ensure that the kubelet file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $kubeConfDir/kubelet; then stat -c %a $kubeConfDir/kubelet; fi"
+      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %a $kubeletconf; fi'"
       tests:
+        bin_op: or
         test_items:
           - flag: "644"
+            compare:
+              op: eq
+              value: 644
+            set: true
+          - flag: "640"
+            compare:
+              op: eq
+              value: "640"
+            set: true
+          - flag: "600"
+            compare:
+              op: eq
+              value: "600"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chmod 644 $kubeConfDir/kubelet"
+              \nFor example, chmod 644 $kubeletconf"
       scored: true
 
     - id: 2.2.4
       text: "Ensure that the kubelet file ownership is set to root:root (Scored)"
-      audit: "if test -e $kubeConfDir/kubelet; then stat -c %U:%G $kubeConfDir/kubelet; fi"
+      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %U:%G $kubeletconf; fi'"
       tests:
         test_items:
           - flag: "root:root"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chown root:root $kubeConfDir/kubelet"
+              \nFor example, chown root:root $kubeletconf"
       scored: true
 
     - id: 2.2.5
       text: "Ensure that the proxy file permissions are set to 644 or more restrictive (Scored)"
-      audit: "if test -e $kubeConfDir/proxy; then stat -c %a $kubeConfDir/proxy; fi"
+      audit: "/bin/sh -c 'if test -e $proxyconf; then stat -c %a $proxyconf; fi'"
       tests:
+        bin_op: or
         test_items:
           - flag: "644"
+            compare:
+              op: eq
+              value: "644"
+            set: true
+          - flag: "640"
+            compare:
+              op: eq
+              value: "640"
+            set: true
+          - flag: "600"
+            compare:
+              op: eq
+              value: "600"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chmod 644 $kubeConfDir/proxy"
+              \nFor example, chmod 644 $proxyconf"
       scored: true
 
     - id: 2.2.6
       text: "Ensure that the proxy file ownership is set to root:root (Scored)"
-      audit: "if test -e $kubeConfDir/proxy; then stat -c %U:%G $kubeConfDir/proxy; fi"
+      audit: "/bin/sh -c 'if test -e $proxyconf; then stat -c %U:%G $proxyconf; fi'"
       tests:
         test_items:
           - flag: "root:root"
             set: true
       remediation: "Run the below command (based on the file location on your system) on the each worker node. 
-              \nFor example, chown root:root $kubeConfDir/proxy"
+              \nFor example, chown root:root $proxyconf"
+      scored: true
+
+    - id: 2.2.7
+      text: "Ensure that the certificate authorities file permissions are set to
+              644 or more restrictive (Scored)"
+      audit: "/bin/sh -c 'if test -e $ca-file; then stat -c %a $ca-file; fi'"
+      tests:
+        bin_op: or
+        test_items:
+          - flag: "644"
+            compare:
+              op: eq
+              value: "644"
+            set: true
+          - flag: "640"
+            compare:
+              op: eq
+              value: "640"
+            set: true
+          - flag: "600"
+            compare:
+              op: eq
+              value: "600"
+            set: true
+      remediation: "Run the following command to modify the file permissions of the --client-ca-file
+              \nchmod 644 <filename>"
+      scored: true
+
+    - id: 2.2.8
+      text: "Ensure that the client certificate authorities file ownership is set to root:root"
+      audit: "/bin/sh -c 'if test -e $ca-file; then stat -c %U:%G $ca-file; fi'"
+      tests:
+        test_items:
+          - flag: "notexist:notexist"
+            set: true
+      remediation: "Run the following command to modify the ownership of the --client-ca-file.
+              \nchown root:root <filename>"
       scored: true

check/check.go

@@ -15,14 +15,15 @@
 package check
 
 import (
-	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"os"
 	"os/exec"
+	"regexp"
 	"strings"
 
-	"github.com/joncalhoun/pipe"
+	"github.com/golang/glog"
 )
 
 // NodeType indicates the type of node (master, node, federated).
@@ -49,12 +50,20 @@ const (
 	FEDERATED NodeType = "federated"
 )
 
+func handleError(err error, context string) (errmsg string) {
+	if err != nil {
+		errmsg = fmt.Sprintf("%s, error: %s\n", context, err)
+	}
+	return
+}
+
 // Check contains information about a recommendation in the
 // CIS Kubernetes 1.6+ document.
 type Check struct {
 	ID          string `yaml:"id" json:"id"`
 	Text        string
 	Audit       string      `json:"omit"`
+	Type        string      `json:"type"`
 	Commands    []*exec.Cmd `json:"omit"`
 	Tests       *tests      `json:"omit"`
 	Set         bool        `json:"omit"`
@@ -65,59 +74,91 @@ type Check struct {
 // Run executes the audit commands specified in a check and outputs
 // the results.
 func (c *Check) Run() {
-	var out string
+	// If check type is manual, force result to WARN.
+	if c.Type == "manual" {
+		c.State = WARN
+		return
+	}
+
+	var out bytes.Buffer
+	var errmsgs string
 
 	// Check if command exists or exit with WARN.
 	for _, cmd := range c.Commands {
-		_, err := exec.LookPath(cmd.Path)
-		if err != nil {
+		if !isShellCommand(cmd.Path) {
 			c.State = WARN
 			return
 		}
 	}
 
 	// Run commands.
-	if len(c.Commands) == 0 {
+	n := len(c.Commands)
+	if n == 0 {
 		// Likely a warning message.
 		c.State = WARN
 		return
 	}
 
-	p, err := pipe.New(c.Commands...)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "init: error creating command pipeline %s\n", err)
-		os.Exit(1)
+	// Each command runs,
+	//   cmd0 out -> cmd1 in, cmd1 out -> cmd2 in ... cmdn out -> os.stdout
+	//   cmd0 err should terminate chain
+	cs := c.Commands
+
+	// Initialize command pipeline
+	cs[n-1].Stdout = &out
+	i := 1
+
+	var err error
+	errmsgs = ""
+
+	for i < n {
+		cs[i-1].Stdout, err = cs[i].StdinPipe()
+		errmsgs += handleError(
+			err,
+			fmt.Sprintf("failed to run: %s\nfailed command: %s",
+				c.Audit,
+				cs[i].Args,
+			),
+		)
+		i++
 	}
 
-	pr, pw := io.Pipe()
-	p.Stdout = pw
-	defer pw.Close()
-
-	if err := p.Start(); err != nil {
-		fmt.Fprintf(os.Stderr, "start: error running audit command %s\n", err)
-		os.Exit(1)
+	// Start command pipeline
+	i = 0
+	for i < n {
+		err := cs[i].Start()
+		errmsgs += handleError(
+			err,
+			fmt.Sprintf("failed to run: %s\nfailed command: %s",
+				c.Audit,
+				cs[i].Args,
+			),
+		)
+		i++
 	}
 
-	// Read output of command chain into string for check.
-	go func() {
-		defer pr.Close()
-		scanner := bufio.NewScanner(pr)
-		for scanner.Scan() {
-			out += scanner.Text()
+	// Complete command pipeline
+	i = 0
+	for i < n {
+		err := cs[i].Wait()
+		errmsgs += handleError(
+			err,
+			fmt.Sprintf("failed to run: %s\nfailed command:%s",
+				c.Audit,
+				cs[i].Args,
+			),
+		)
+
+		if i < n-1 {
+			cs[i].Stdout.(io.Closer).Close()
 		}
 
-		if err := scanner.Err(); err != nil {
-			fmt.Fprintf(os.Stderr, "error accumulating  output %s\n", err)
-			os.Exit(1)
-		}
-	}()
-
-	if err := p.Wait(); err != nil {
-		fmt.Fprintf(os.Stderr, "wait: error running audit command %s\n", err)
-		os.Exit(1)
+		i++
 	}
 
-	res := c.Tests.execute(out)
+	glog.V(2).Info("%s\n", errmsgs)
+
+	res := c.Tests.execute(out.String())
 	if res {
 		c.State = PASS
 	} else {
@@ -125,18 +166,44 @@ func (c *Check) Run() {
 	}
 }
 
-// textToCommand transforms a text representation of commands to be
+// textToCommand transforms an input text representation of commands to be
 // run into a slice of commands.
 // TODO: Make this more robust.
 func textToCommand(s string) []*exec.Cmd {
 	cmds := []*exec.Cmd{}
 
 	cp := strings.Split(s, "|")
-	// fmt.Println("check.toCommand:", cp)
 
 	for _, v := range cp {
 		v = strings.Trim(v, " ")
-		cs := strings.Split(v, " ")
+
+		// TODO:
+		// GOAL: To split input text into arguments for exec.Cmd.
+		//
+		// CHALLENGE: The input text may contain quoted strings that
+		// must be passed as a unit to exec.Cmd.
+		// eg. bash -c 'foo bar'
+		// 'foo bar' must be passed as unit to exec.Cmd if not the command
+		// will fail when it is executed.
+		// eg. exec.Cmd("bash", "-c", "foo bar")
+		//
+		// PROBLEM: Current solution assumes the grouped string will always
+		// be at the end of the input text.
+		re := regexp.MustCompile(`^(.*)(['"].*['"])$`)
+		grps := re.FindStringSubmatch(v)
+
+		var cs []string
+		if len(grps) > 0 {
+			s := strings.Trim(grps[1], " ")
+			cs = strings.Split(s, " ")
+
+			s1 := grps[len(grps)-1]
+			s1 = strings.Trim(s1, "'\"")
+
+			cs = append(cs, s1)
+		} else {
+			cs = strings.Split(v, " ")
+		}
 
 		cmd := exec.Command(cs[0], cs[1:]...)
 		cmds = append(cmds, cmd)
@@ -144,3 +211,18 @@ func textToCommand(s string) []*exec.Cmd {
 
 	return cmds
 }
+
+func isShellCommand(s string) bool {
+	cmd := exec.Command("/bin/sh", "-c", "command -v "+s)
+
+	out, err := cmd.Output()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s\n", err)
+		os.Exit(1)
+	}
+
+	if strings.Contains(string(out), s) {
+		return true
+	}
+	return false
+}

check/controls.go

@@ -17,7 +17,6 @@ package check
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 
 	yaml "gopkg.in/yaml.v2"
 )
@@ -46,19 +45,16 @@ type Summary struct {
 }
 
 // NewControls instantiates a new master Controls object.
-func NewControls(t NodeType, in []byte) *Controls {
-	var err error
+func NewControls(t NodeType, in []byte) (*Controls, error) {
 	c := new(Controls)
 
-	err = yaml.Unmarshal(in, c)
+	err := yaml.Unmarshal(in, c)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
-		os.Exit(1)
+		return nil, fmt.Errorf("failed to unmarshal YAML: %s", err)
 	}
 
 	if t != c.Type {
-		fmt.Fprintf(os.Stderr, "non-%s controls file specified\n", t)
-		os.Exit(1)
+		return nil, fmt.Errorf("non-%s controls file specified", t)
 	}
 
 	// Prepare audit commands
@@ -68,7 +64,7 @@ func NewControls(t NodeType, in []byte) *Controls {
 		}
 	}
 
-	return c
+	return c, nil
 }
 
 // RunGroup runs all checks in a group.

check/controls_test.go

@@ -0,0 +1,32 @@
+package check
+
+import (
+	"io/ioutil"
+	"testing"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+const cfgDir = "../cfg/"
+
+// validate that the files we're shipping are valid YAML
+func TestYamlFiles(t *testing.T) {
+	files, err := ioutil.ReadDir(cfgDir)
+	if err != nil {
+		t.Fatalf("error reading %s directory: %v", cfgDir, err)
+	}
+	for _, file := range files {
+		fileName := file.Name()
+		in, err := ioutil.ReadFile(cfgDir + fileName)
+		if err != nil {
+			t.Fatalf("error opening file %s: %v", fileName, err)
+		}
+
+		c := new(Controls)
+
+		err = yaml.Unmarshal(in, c)
+		if err != nil {
+			t.Fatalf("failed to load YAML from %s: %v", fileName, err)
+		}
+	}
+}

check/data

@@ -7,59 +7,42 @@ groups:
 - id: 1.1
   text: "Kube-apiserver"
   checks:
-    - id: 1.1.1
-      text: "Ensure that the --allow-privileged argument is set (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+    - id: 0
+      text: "flag is set"
       tests:
         test_items:
-          -
-            flag: "--allow-privileged"
+          - flag: "--allow-privileged"
             set: true
-      remediation: "Edit the /etc/kubernetes/config file on the master node and set the KUBE_ALLOW_PRIV parameter to '--allow-privileged=false'"
-      scored: true
 
-    - id: 1.1.2
-      text: "Ensure that the --basic-auth argument is not set (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+    - id: 1
+      text: "flag is not set"
       tests:
         test_item:
-          -
-            flag: "--basic-auth"
+          - flag: "--basic-auth"
             set: false
-      remediation: "Edit the /etc/kubernetes/config file on the master node and set the KUBE_ALLOW_PRIV parameter to '--allow-privileged=false'"
-      scored: true
 
-    - id: 1.1.3
-      text: "Ensure that the --insecure-port argument is set to 0 (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+    - id: 2
+      text: "flag value is set to some value"
       tests:
         test_items:
-          -
-            flag: "--insecure-port"
+          - flag: "--insecure-port"
             compare:
               op: eq
               value: 0
             set: true
-      remediation: "Edit the /etc/kubernetes/config file on the master node and set the KUBE_ALLOW_PRIV parameter to '--allow-privileged=false'"
-      scored: true
 
-    - id: 1.1.4
-      text: "Ensure that the --audit-log-maxage argument is set to 30 or appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+    - id: 3
+      text: "flag value is greater than or equal some number"
       tests:
         test_items:
-          -
-            flag: "--audit-log-maxage"
+          - flag: "--audit-log-maxage"
             compare:
               op: gte
               value: 30
             set: true
-      remediation: "Edit the /etc/kubernetes/config file on the master node and set the KUBE_ALLOW_PRIV parameter to '--allow-privileged=false'"
-      scored: true
 
-    - id: 1.1.5
-      text: "Ensure that the --max-backlog argument is set to 30 or less (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+    - id: 4
+      text: "flag value is less than some number"
       tests:
         test_items:
           - flag: "--max-backlog"
@@ -67,26 +50,19 @@ groups:
               op: lt
               value: 30
             set: true
-      remediation: "Edit the /etc/kubernetes/config file on the master node and set the KUBE_ALLOW_PRIV parameter to '--allow-privileged=false'"
-      scored: true
 
-    - id: 1.1.6
-      text: "Ensure admission control does not include AlwaysAdmit (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+    - id: 5
+      text: "flag value does not have some value"
       tests:
         test_items:
-          -
-            flag: "--admission-control"
+          - flag: "--admission-control"
             compare:
               op: nothave
               value: AlwaysAdmit
             set: true
-      remediation: "Edit the /etc/kubernetes/config file on the master node and set the KUBE_ALLOW_PRIV parameter to '--allow-privileged=false'"
-      scored: true
 
-    - id: 1.1.7
-      text: "Ensure that the --kubelet-client-certificate and --kubelet-clientkey arguments are set as appropriate (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+    - id: 6
+      text: "test AND binary operation"
       tests:
         bin_op: and
         test_items:
@@ -94,17 +70,13 @@ groups:
           set: true
         - flag: "--kubelet-clientkey"
           set: true
-      remediation: "Edit the /etc/kubernetes/config file on the master node and set the KUBE_ALLOW_PRIV parameter to '--allow-privileged=false'"
-      scored: true
 
-    - id: 1.1.8
-      text: "Ensure that the --secure-port argument is not set to 0 (Scored)"
-      audit: "ps -ef | grep kube-apiserver | grep -v grep"
+    - id: 7
+      text: "test OR binary operation"
       tests:
         bin_op: or
         test_items:
-          -
-            flag:  "--secure-port"
+          - flag:  "--secure-port"
             compare:
               op: eq
               value: 0
@@ -112,15 +84,35 @@ groups:
           -
             flag: "--secure-port"
             set: false
-      remediation: "Edit the /etc/kubernetes/apiserver file on the master node and either remove the -secure-port argument from the KUBE_API_ARGS parameter or set it to a different desired port."
-      scored: true
 
-    - id: 1.4.1
-      text: "Ensure that the apiserver file permissions are set to 644 or more restrictive (Scored)"
-      audit: "stat -c %a /etc/kubernetes/apiserver"
+    - id: 8
+      text: "test flag with arbitrary text"
       tests:
         test_items:
         - flag: "644"
+          compare:
+            op: eq
+            value: "644"
+          set: true
+
+    - id: 9
+      text: "test permissions"
+      audit: "/bin/sh -c 'if test -e $config; then stat -c %a $config; fi'"
+      tests:
+        bin_op: or
+        test_items:
+        - flag: "644"
+          compare:
+            op: eq
+            value: "644"
+          set: true
+        - flag: "640"
+          compare:
+            op: eq
+            value: "640"
+          set: true
+        - flag: "600"
+          compare:
+            op: eq
+            value: "600"
           set: true
-      remediation: "Run the below command (based on the file location on your system) on the master node. For example, chmod 644 /etc/kubernetes/apiserver"
-      scored: true

check/test.go

@@ -38,6 +38,7 @@ const (
 
 type testItem struct {
 	Flag    string
+	Output  string
 	Value   string
 	Set     bool
 	Compare compare
@@ -57,14 +58,22 @@ func (t *testItem) execute(s string) (result bool) {
 		isset := match
 
 		if isset && t.Compare.Op != "" {
-			pttn := t.Flag + `=(\S*) *`
+			// Expects flags in the form;
+			// --flag=somevalue
+			// --flag
+			// somevalue
+			pttn := `(` + t.Flag + `)(=)*([^\s,]*) *`
 			flagRe := regexp.MustCompile(pttn)
 			vals := flagRe.FindStringSubmatch(s)
 
 			if len(vals) > 0 {
-				flagVal = vals[1]
+				if vals[3] != "" {
+					flagVal = vals[3]
+				} else {
+					flagVal = vals[1]
+				}
 			} else {
-				fmt.Fprintf(os.Stderr, "expected value for %s but none found\n", t.Flag)
+				fmt.Fprintf(os.Stderr, "invalid flag in testitem definition")
 				os.Exit(1)
 			}
 

check/test_test.go

@@ -16,6 +16,8 @@ package check
 
 import (
 	"io/ioutil"
+	"os"
+	"strings"
 	"testing"
 )
 
@@ -30,66 +32,74 @@ func init() {
 	if err != nil {
 		panic("Failed reading test data: " + err.Error())
 	}
-	controls = NewControls(MASTER, in)
+
+	// substitute variables in data file
+	user := os.Getenv("USER")
+	s := strings.Replace(string(in), "$user", user, -1)
+
+	controls, err = NewControls(MASTER, []byte(s))
+	// controls, err = NewControls(MASTER, in)
+	if err != nil {
+		panic("Failed creating test controls: " + err.Error())
+	}
 }
 
 func TestTestExecute(t *testing.T) {
+
 	cases := []struct {
-		*tests
-		testfor string
-		str     string
+		*Check
+		str string
 	}{
 		{
-			controls.Groups[0].Checks[0].Tests,
-			"flag set",
+			controls.Groups[0].Checks[0],
 			"2:45 ../kubernetes/kube-apiserver --allow-privileged=false --option1=20,30,40",
 		},
 		{
-			controls.Groups[0].Checks[1].Tests,
-			"flag not set",
+			controls.Groups[0].Checks[1],
 			"2:45 ../kubernetes/kube-apiserver --allow-privileged=false",
 		},
 		{
-			controls.Groups[0].Checks[2].Tests,
-			"flag and value set",
+			controls.Groups[0].Checks[2],
 			"niinai   13617  2635 99 19:26 pts/20   00:03:08 ./kube-apiserver --insecure-port=0 --anonymous-auth",
 		},
 		{
-			controls.Groups[0].Checks[3].Tests,
-			"flag value greater than value",
+			controls.Groups[0].Checks[3],
 			"2:45 ../kubernetes/kube-apiserver --secure-port=0 --audit-log-maxage=40 --option",
 		},
 		{
-			controls.Groups[0].Checks[4].Tests,
-			"flag value less than value",
+			controls.Groups[0].Checks[4],
 			"2:45 ../kubernetes/kube-apiserver --max-backlog=20 --secure-port=0 --audit-log-maxage=40 --option",
 		},
 		{
-			controls.Groups[0].Checks[5].Tests,
-			"flag value does not have",
+			controls.Groups[0].Checks[5],
 			"2:45 ../kubernetes/kube-apiserver --option --admission-control=WebHook,RBAC ---audit-log-maxage=40",
 		},
 		{
-			controls.Groups[0].Checks[6].Tests,
-			"AND multiple tests, all testitems pass",
+			controls.Groups[0].Checks[6],
 			"2:45 .. --kubelet-clientkey=foo --kubelet-client-certificate=bar --admission-control=Webhook,RBAC",
 		},
 		{
-			controls.Groups[0].Checks[7].Tests,
-			"OR multiple tests",
+			controls.Groups[0].Checks[7],
 			"2:45 ..  --secure-port=0 --kubelet-client-certificate=bar --admission-control=Webhook,RBAC",
 		},
 		{
-			controls.Groups[0].Checks[8].Tests,
-			"text",
+			controls.Groups[0].Checks[8],
 			"644",
 		},
+		{
+			controls.Groups[0].Checks[9],
+			"640",
+		},
+		{
+			controls.Groups[0].Checks[9],
+			"600",
+		},
 	}
 
 	for _, c := range cases {
-		res := c.tests.execute(c.str)
+		res := c.Tests.execute(c.str)
 		if !res {
-			t.Errorf("%s, expected:%v, got:%v\n", c.testfor, true, res)
+			t.Errorf("%s, expected:%v, got:%v\n", c.Text, true, res)
 		}
 	}
 }

cmd/common.go

@@ -18,54 +18,67 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
-	"regexp"
-	"strings"
 
 	"github.com/aquasecurity/kube-bench/check"
-	"github.com/fatih/color"
 	"github.com/spf13/viper"
 )
 
 var (
-	kubeMasterBin  = []string{"kube-apiserver", "kube-scheduler", "kube-controller-manager"}
-	kubeMasterConf = []string{}
+	apiserverBin            string
+	apiserverConf           string
+	schedulerBin            string
+	schedulerConf           string
+	controllerManagerBin    string
+	controllerManagerConf   string
+	config                  string
+	etcdBin                 string
+	etcdConf                string
+	flanneldBin             string
+	flanneldConf            string
+	kubeletBin              string
+	kubeletConf             string
+	proxyBin                string
+	proxyConf               string
+	fedApiserverBin         string
+	fedControllerManagerBin string
 
-	kubeNodeBin  = []string{"kubelet"}
-	kubeNodeConf = []string{}
-
-	kubeFederatedBin  = []string{"federation-apiserver", "federation-controller-manager"}
-	kubeFederatedConf = []string{}
+	errmsgs string
 
 	// TODO: Consider specifying this in config file.
-	kubeVersion = "Kubernetes v1.6"
-
-	// Used for variable substitution
-	symbols = map[string]string{}
-
-	// Print colors
-	colors = map[check.State]*color.Color{
-		check.PASS: color.New(color.FgGreen),
-		check.FAIL: color.New(color.FgRed),
-		check.WARN: color.New(color.FgYellow),
-		check.INFO: color.New(color.FgBlue),
-	}
+	kubeMajorVersion = "1"
+	kubeMinorVersion = "7"
 )
 
 func runChecks(t check.NodeType) {
 	var summary check.Summary
 	var file string
 
-	// Set up for config file check.
-	kubeMasterConf = append(kubeMasterConf, viper.Get("kubeConfDir").(string)+"/apiserver")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("kubeConfDir").(string)+"/scheduler")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("kubeConfDir").(string)+"/controller-manager")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("kubeConfDir").(string)+"/config")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("etcdConfDir").(string)+"/etcd.conf")
-	kubeMasterConf = append(kubeMasterConf, viper.Get("flanneldConfDir").(string)+"/flanneld")
-	kubeNodeConf = append(kubeNodeConf, viper.Get("kubeConfDir").(string)+"/kubelet")
-	kubeNodeConf = append(kubeNodeConf, viper.Get("kubeConfDir").(string)+"/proxy")
+	// Master variables
+	apiserverBin = viper.GetString("installation." + installation + ".master.bin.apiserver")
+	apiserverConf = viper.GetString("installation." + installation + ".master.conf.apiserver")
+	schedulerBin = viper.GetString("installation." + installation + ".master.bin.scheduler")
+	schedulerConf = viper.GetString("installation." + installation + ".master.conf.scheduler")
+	controllerManagerBin = viper.GetString("installation." + installation + ".master.bin.controller-manager")
+	controllerManagerConf = viper.GetString("installation." + installation + ".master.conf.controller-manager")
+	config = viper.GetString("installation." + installation + ".config")
 
+	etcdBin = viper.GetString("etcd.bin")
+	etcdConf = viper.GetString("etcd.conf")
+	flanneldBin = viper.GetString("flanneld.bin")
+	flanneldConf = viper.GetString("flanneld.conf")
+
+	// Node variables
+	kubeletBin = viper.GetString("installation." + installation + ".node.bin.kubelet")
+	kubeletConf = viper.GetString("installation." + installation + ".node.conf.kubelet")
+	proxyBin = viper.GetString("installation." + installation + ".node.bin.proxy")
+	proxyConf = viper.GetString("installation." + installation + ".node.conf.proxy")
+
+	// Federated
+	fedApiserverBin = viper.GetString("installation." + installation + ".federated.bin.apiserver")
+	fedControllerManagerBin = viper.GetString("installation." + installation + ".federated.bin.controller-manager")
+
+	// Run kubernetes installation validation checks.
+	verifyKubeVersion(kubeMajorVersion, kubeMinorVersion)
 	verifyNodeType(t)
 
 	switch t {
@@ -79,39 +92,53 @@ func runChecks(t check.NodeType) {
 
 	in, err := ioutil.ReadFile(file)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "error opening %s controls file: %s\n", t, err)
-		os.Exit(1)
+		exitWithError(fmt.Errorf("error opening %s controls file: %v", t, err))
 	}
 
-	// Variable substitutions. Replace all occurrences of variables in controls file.
-	s := strings.Replace(string(in), "$kubeConfDir", viper.Get("kubeConfDir").(string), -1)
-	s = strings.Replace(s, "$etcdConfDir", viper.Get("etcdConfDir").(string), -1)
-	s = strings.Replace(s, "$flanneldConfDir", viper.Get("etcdConfDir").(string), -1)
+	// Variable substitutions. Replace all occurrences of variables in controls files.
+	s := multiWordReplace(string(in), "$apiserverbin", apiserverBin)
+	s = multiWordReplace(s, "$apiserverconf", apiserverConf)
+	s = multiWordReplace(s, "$schedulerbin", schedulerBin)
+	s = multiWordReplace(s, "$schedulerconf", schedulerConf)
+	s = multiWordReplace(s, "$controllermanagerbin", controllerManagerBin)
+	s = multiWordReplace(s, "$controllermanagerconf", controllerManagerConf)
+	s = multiWordReplace(s, "$config", config)
 
-	controls := check.NewControls(t, []byte(s))
+	s = multiWordReplace(s, "$etcdbin", etcdBin)
+	s = multiWordReplace(s, "$etcdconf", etcdConf)
+	s = multiWordReplace(s, "$flanneldbin", flanneldBin)
+	s = multiWordReplace(s, "$flanneldconf", flanneldConf)
+
+	s = multiWordReplace(s, "$kubeletbin", kubeletBin)
+	s = multiWordReplace(s, "$kubeletconf", kubeletConf)
+	s = multiWordReplace(s, "$proxybin", proxyBin)
+	s = multiWordReplace(s, "$proxyconf", proxyConf)
+
+	s = multiWordReplace(s, "$fedapiserverbin", fedApiserverBin)
+	s = multiWordReplace(s, "$fedcontrollermanagerbin", fedControllerManagerBin)
+
+	controls, err := check.NewControls(t, []byte(s))
+	if err != nil {
+		exitWithError(fmt.Errorf("error setting up %s controls: %v", t, err))
+	}
 
 	if groupList != "" && checkList == "" {
-		// log.Println("group: set, checks: not set")
 		ids := cleanIDs(groupList)
 		summary = controls.RunGroup(ids...)
-
 	} else if checkList != "" && groupList == "" {
-		// log.Println("group: not set, checks: set")
 		ids := cleanIDs(checkList)
 		summary = controls.RunChecks(ids...)
-
 	} else if checkList != "" && groupList != "" {
-		// log.Println("group: set, checks: set")
-		fmt.Fprintf(os.Stderr, "group option and check option can't be used together\n")
-		os.Exit(1)
-
+		exitWithError(fmt.Errorf("group option and check option can't be used together"))
 	} else {
 		summary = controls.RunGroup()
 	}
 
-	if jsonFmt {
+	// if we successfully ran some tests and it's json format, ignore the warnings
+	if (summary.Fail > 0 || summary.Warn > 0 || summary.Pass > 0) && jsonFmt {
 		out, err := controls.JSON()
 		if err != nil {
+			exitWithError(fmt.Errorf("failed to output in JSON format: %v", err))
 		}
 
 		fmt.Println(string(out))
@@ -120,71 +147,37 @@ func runChecks(t check.NodeType) {
 	}
 }
 
-func cleanIDs(list string) []string {
-	list = strings.Trim(list, ",")
-	ids := strings.Split(list, ",")
-
-	for _, id := range ids {
-		id = strings.Trim(id, " ")
-	}
-
-	return ids
-}
-
+// verifyNodeType checks the executables and config files are as expected
+// for the specified tests (master, node or federated).
 func verifyNodeType(t check.NodeType) {
-	var binPath []string
-	var confPath []string
-	var out []byte
+	var bins []string
+	var confs []string
 
 	switch t {
 	case check.MASTER:
-		binPath = kubeMasterBin
-		confPath = kubeMasterConf
+		bins = []string{apiserverBin, schedulerBin, controllerManagerBin}
+		confs = []string{apiserverConf, schedulerConf, controllerManagerConf}
 	case check.NODE:
-		binPath = kubeNodeBin
-		confPath = kubeNodeConf
+		bins = []string{kubeletBin, proxyBin}
+		confs = []string{kubeletConf, proxyConf}
 	case check.FEDERATED:
-		binPath = kubeFederatedBin
-		confPath = kubeFederatedConf
+		bins = []string{fedApiserverBin, fedControllerManagerBin}
 	}
 
-	// These executables might not be on the user's path.
-	// TODO! Check the version number using kubectl, which is more likely to be on the path.
-	for _, b := range binPath {
-		_, err := exec.LookPath(b)
+	for _, bin := range bins {
+		if !verifyBin(bin, ps) {
+			printlnWarn(fmt.Sprintf("%s is not running", bin))
+		}
+	}
+
+	for _, conf := range confs {
+		_, err := os.Stat(conf)
 		if err != nil {
-			colorPrint(check.WARN, fmt.Sprintf("%s: command not found on path - version check skipped\n", b))
-			continue
-		}
-
-		// Check version
-		cmd := exec.Command(b, "--version")
-		out, _ = cmd.Output()
-		if matched, _ := regexp.MatchString(kubeVersion, string(out)); !matched {
-			colorPrint(check.FAIL,
-				fmt.Sprintf(
-					"%s unsupported version, expected %s, got %s\n",
-					b,
-					kubeVersion,
-					string(out),
-				))
-			os.Exit(1)
-		}
-	}
-
-	for _, b := range binPath {
-		// Check if running.
-		cmd := exec.Command("ps", "-ef")
-		out, _ = cmd.Output()
-		if matched, _ := regexp.MatchString(".*"+b, string(out)); !matched {
-			colorPrint(check.FAIL, fmt.Sprintf("%s is not running\n", b))
-			os.Exit(1)
-		}
-	}
-
-	for _, c := range confPath {
-		if _, err := os.Stat(c); os.IsNotExist(err) {
-			colorPrint(check.WARN, fmt.Sprintf("config file %s does not exist\n", c))
+			if os.IsNotExist(err) {
+				printlnWarn(fmt.Sprintf("Missing kubernetes config file: %s", conf))
+			} else {
+				exitWithError(fmt.Errorf("error looking for file %s: %v", conf, err))
+			}
 		}
 	}
 }
@@ -195,8 +188,10 @@ func colorPrint(state check.State, s string) {
 	fmt.Printf("%s", s)
 }
 
+// prettyPrint outputs the results to stdout in human-readable format
 func prettyPrint(r *check.Controls, summary check.Summary) {
-	// Print checks and results.
+	colorPrint(check.INFO, fmt.Sprintf("Using config file: %s\n", viper.ConfigFileUsed()))
+
 	colorPrint(check.INFO, fmt.Sprintf("%s %s\n", r.ID, r.Text))
 	for _, g := range r.Groups {
 		colorPrint(check.INFO, fmt.Sprintf("%s %s\n", g.ID, g.Text))

cmd/root.go

@@ -15,6 +15,7 @@
 package cmd
 
 import (
+	goflag "flag"
 	"fmt"
 	"os"
 
@@ -34,9 +35,13 @@ var (
 	nodeFile      string
 	federatedFile string
 
-	kubeConfDir    string
-	etcdConfDir    string
-	flannelConfDir string
+	loud bool
+
+	kubeConfDir     string
+	etcdConfDir     string
+	flanneldConfDir string
+
+	installation string
 )
 
 // RootCmd represents the base command when called without any subcommands
@@ -49,6 +54,9 @@ var RootCmd = &cobra.Command{
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
+	goflag.Set("logtostderr", "true")
+	goflag.CommandLine.Parse([]string{})
+
 	if err := RootCmd.Execute(); err != nil {
 		fmt.Println(err)
 		os.Exit(-1)
@@ -59,19 +67,32 @@ func init() {
 	cobra.OnInitialize(initConfig)
 
 	RootCmd.PersistentFlags().BoolVar(&jsonFmt, "json", false, "Prints the results as JSON")
-	RootCmd.PersistentFlags().StringVarP(&checkList,
+	RootCmd.PersistentFlags().StringVar(
+		&installation,
+		"installation",
+		"default",
+		"Specify how kubernetes cluster was installed. Possible values are default,hyperkube,kops,kubeadm",
+	)
+	RootCmd.PersistentFlags().StringVarP(
+		&checkList,
 		"check",
 		"c",
 		"",
 		`A comma-delimited list of checks to run as specified in CIS document. Example --check="1.1.1,1.1.2"`,
 	)
-	RootCmd.PersistentFlags().StringVarP(&groupList,
+	RootCmd.PersistentFlags().StringVarP(
+		&groupList,
 		"group",
 		"g",
 		"",
 		`Run all the checks under this comma-delimited list of groups. Example --group="1.1"`,
 	)
 	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is ./cfg/config.yaml)")
+
+	goflag.CommandLine.VisitAll(func(goflag *goflag.Flag) {
+		RootCmd.PersistentFlags().AddGoFlag(goflag)
+	})
+
 }
 
 // initConfig reads in config file and ENV variables if set.
@@ -83,23 +104,12 @@ func initConfig() {
 		viper.AddConfigPath(cfgDir)   // adding ./cfg as first search path
 	}
 
-	viper.SetEnvPrefix("CISK8S")
+	viper.SetEnvPrefix("KUBE_BENCH")
 	viper.AutomaticEnv() // read in environment variables that match
 
-	// Set defaults
-	viper.SetDefault("kubeConfDir", "/etc/kubernetes")
-	viper.SetDefault("etcdConfDir", "/etc/etcd")
-	viper.SetDefault("flanneldConfDir", "/etc/sysconfig")
-
-	viper.SetDefault("masterFile", cfgDir+"/master.yaml")
-	viper.SetDefault("nodeFile", cfgDir+"/node.yaml")
-	viper.SetDefault("federatedFile", cfgDir+"/federated.yaml")
-
 	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		colorPrint(check.INFO, fmt.Sprintf("Using config file: %s\n", viper.ConfigFileUsed()))
-	} else {
-		colorPrint(check.FAIL, fmt.Sprintf("Faied to read config file: %v\n", err))
+	if err := viper.ReadInConfig(); err != nil {
+		colorPrint(check.FAIL, fmt.Sprintf("Failed to read config file: %v\n", err))
 		os.Exit(1)
 	}
 }

cmd/util.go

@@ -0,0 +1,161 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+
+	"github.com/aquasecurity/kube-bench/check"
+	"github.com/fatih/color"
+	"github.com/golang/glog"
+)
+
+var (
+	// Print colors
+	colors = map[check.State]*color.Color{
+		check.PASS: color.New(color.FgGreen),
+		check.FAIL: color.New(color.FgRed),
+		check.WARN: color.New(color.FgYellow),
+		check.INFO: color.New(color.FgBlue),
+	}
+)
+
+func printlnWarn(msg string) {
+	fmt.Fprintf(os.Stderr, "[%s] %s\n",
+		colors[check.WARN].Sprintf("%s", check.WARN),
+		msg,
+	)
+}
+
+func sprintlnWarn(msg string) string {
+	return fmt.Sprintf("[%s] %s",
+		colors[check.WARN].Sprintf("%s", check.WARN),
+		msg,
+	)
+}
+
+func exitWithError(err error) {
+	fmt.Fprintf(os.Stderr, "\n%v\n", err)
+	os.Exit(1)
+}
+
+func continueWithError(err error, msg string) string {
+	if err != nil {
+		glog.V(1).Info(err)
+	}
+
+	if msg != "" {
+		fmt.Fprintf(os.Stderr, "%s\n", msg)
+	}
+
+	return ""
+}
+
+func cleanIDs(list string) []string {
+	list = strings.Trim(list, ",")
+	ids := strings.Split(list, ",")
+
+	for _, id := range ids {
+		id = strings.Trim(id, " ")
+	}
+
+	return ids
+}
+
+// ps execs out to the ps command; it's separated into a function so we can write tests
+func ps(proc string) string {
+	cmd := exec.Command("ps", "-C", proc, "-o", "cmd", "--no-headers")
+	out, err := cmd.Output()
+	if err != nil {
+		continueWithError(fmt.Errorf("%s: %s", cmd.Args, err), "")
+	}
+
+	return string(out)
+}
+
+// verifyBin checks that the binary specified is running
+func verifyBin(bin string, psFunc func(string) string) bool {
+
+	// Strip any quotes
+	bin = strings.Trim(bin, "'\"")
+
+	// bin could consist of more than one word
+	// We'll search for running processes with the first word, and then check the whole
+	// proc as supplied is included in the results
+	proc := strings.Fields(bin)[0]
+	out := psFunc(proc)
+
+	return strings.Contains(out, bin)
+}
+
+func verifyKubeVersion(major string, minor string) {
+	// These executables might not be on the user's path.
+
+	_, err := exec.LookPath("kubectl")
+	if err != nil {
+		continueWithError(err, sprintlnWarn("Kubernetes version check skipped"))
+		return
+	}
+
+	cmd := exec.Command("kubectl", "version")
+	out, err := cmd.Output()
+	if err != nil {
+		s := fmt.Sprintf("Kubernetes version check skipped with error %v", err)
+		continueWithError(err, sprintlnWarn(s))
+		if len(out) == 0 {
+			return
+		}
+	}
+
+	msg := checkVersion("Client", string(out), major, minor)
+	if msg != "" {
+		continueWithError(fmt.Errorf(msg), msg)
+	}
+
+	msg = checkVersion("Server", string(out), major, minor)
+	if msg != "" {
+		continueWithError(fmt.Errorf(msg), msg)
+	}
+}
+
+var regexVersionMajor = regexp.MustCompile("Major:\"([0-9]+)\"")
+var regexVersionMinor = regexp.MustCompile("Minor:\"([0-9]+)\"")
+
+func checkVersion(x string, s string, expMajor string, expMinor string) string {
+	regexVersion, err := regexp.Compile(x + " Version: version.Info{(.*)}")
+	if err != nil {
+		return fmt.Sprintf("Error checking Kubernetes version: %v", err)
+	}
+
+	ss := regexVersion.FindString(s)
+	major := versionMatch(regexVersionMajor, ss)
+	minor := versionMatch(regexVersionMinor, ss)
+	if major == "" || minor == "" {
+		return fmt.Sprintf("Couldn't find %s version from kubectl output '%s'", x, s)
+	}
+
+	if major != expMajor || minor != expMinor {
+		return fmt.Sprintf("Unexpected %s version %s.%s", x, major, minor)
+	}
+
+	return ""
+}
+
+func versionMatch(r *regexp.Regexp, s string) string {
+	match := r.FindStringSubmatch(s)
+	if len(match) < 2 {
+		return ""
+	}
+	return match[1]
+}
+
+func multiWordReplace(s string, subname string, sub string) string {
+	f := strings.Fields(sub)
+	if len(f) > 1 {
+		sub = "'" + sub + "'"
+	}
+
+	return strings.Replace(s, subname, sub, -1)
+}

cmd/util_test.go

@@ -0,0 +1,131 @@
+// Copyright © 2017 Aqua Security Software Ltd. <info@aquasec.com>
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"regexp"
+	"strconv"
+	"testing"
+)
+
+func TestCheckVersion(t *testing.T) {
+	kubeoutput := `Client Version: version.Info{Major:"1", Minor:"7", GitVersion:"v1.7.0", GitCommit:"d3ada0119e776222f11ec7945e6d860061339aad", GitTreeState:"clean", BuildDate:"2017-06-30T09:51:01Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"darwin/amd64"}
+	Server Version: version.Info{Major:"1", Minor:"7", GitVersion:"v1.7.0", GitCommit:"d3ada0119e776222f11ec7945e6d860061339aad", GitTreeState:"clean", BuildDate:"2017-07-26T00:12:31Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"}`
+	cases := []struct {
+		t     string
+		s     string
+		major string
+		minor string
+		exp   string
+	}{
+		{t: "Client", s: kubeoutput, major: "1", minor: "7"},
+		{t: "Server", s: kubeoutput, major: "1", minor: "7"},
+		{t: "Client", s: kubeoutput, major: "1", minor: "6", exp: "Unexpected Client version 1.7"},
+		{t: "Client", s: kubeoutput, major: "2", minor: "0", exp: "Unexpected Client version 1.7"},
+		{t: "Server", s: "something unexpected", major: "2", minor: "0", exp: "Couldn't find Server version from kubectl output 'something unexpected'"},
+	}
+
+	for id, c := range cases {
+		t.Run(strconv.Itoa(id), func(t *testing.T) {
+			m := checkVersion(c.t, c.s, c.major, c.minor)
+			if m != c.exp {
+				t.Fatalf("Got: %s, expected: %s", m, c.exp)
+			}
+		})
+	}
+
+}
+
+func TestVersionMatch(t *testing.T) {
+	minor := regexVersionMinor
+	major := regexVersionMajor
+	client := `Client Version: version.Info{Major:"1", Minor:"7", GitVersion:"v1.7.0", GitCommit:"d3ada0119e776222f11ec7945e6d860061339aad", GitTreeState:"clean", BuildDate:"2017-06-30T09:51:01Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"darwin/amd64"}`
+	server := `Server Version: version.Info{Major:"1", Minor:"7", GitVersion:"v1.7.0", GitCommit:"d3ada0119e776222f11ec7945e6d860061339aad", GitTreeState:"clean", BuildDate:"2017-07-26T00:12:31Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"}`
+
+	cases := []struct {
+		r   *regexp.Regexp
+		s   string
+		exp string
+	}{
+		{r: major, s: server, exp: "1"},
+		{r: minor, s: server, exp: "7"},
+		{r: major, s: client, exp: "1"},
+		{r: minor, s: client, exp: "7"},
+		{r: major, s: "Some unexpected string"},
+		{r: minor}, // Checking that we don't fall over if the string is empty
+	}
+
+	for id, c := range cases {
+		t.Run(strconv.Itoa(id), func(t *testing.T) {
+			m := versionMatch(c.r, c.s)
+			if m != c.exp {
+				t.Fatalf("Got %s expected %s", m, c.exp)
+			}
+		})
+	}
+}
+
+var g string
+
+func fakeps(proc string) string {
+	return g
+}
+func TestVerifyBin(t *testing.T) {
+	cases := []struct {
+		proc  string
+		psOut string
+		exp   bool
+	}{
+		{proc: "single", psOut: "single", exp: true},
+		{proc: "single", psOut: "", exp: false},
+		{proc: "two words", psOut: "two words", exp: true},
+		{proc: "two words", psOut: "", exp: false},
+		{proc: "cmd", psOut: "cmd param1 param2", exp: true},
+		{proc: "cmd param", psOut: "cmd param1 param2", exp: true},
+		{proc: "cmd param", psOut: "cmd", exp: false},
+	}
+
+	for id, c := range cases {
+		t.Run(strconv.Itoa(id), func(t *testing.T) {
+			g = c.psOut
+			v := verifyBin(c.proc, fakeps)
+			if v != c.exp {
+				t.Fatalf("Expected %v got %v", c.exp, v)
+			}
+		})
+	}
+}
+
+func TestMultiWordReplace(t *testing.T) {
+	cases := []struct {
+		input   string
+		sub     string
+		subname string
+		output  string
+	}{
+		{input: "Here's a file with no substitutions", sub: "blah", subname: "blah", output: "Here's a file with no substitutions"},
+		{input: "Here's a file with a substitution", sub: "blah", subname: "substitution", output: "Here's a file with a blah"},
+		{input: "Here's a file with multi-word substitutions", sub: "multi word", subname: "multi-word", output: "Here's a file with 'multi word' substitutions"},
+		{input: "Here's a file with several several substitutions several", sub: "blah", subname: "several", output: "Here's a file with blah blah substitutions blah"},
+	}
+	for id, c := range cases {
+		t.Run(strconv.Itoa(id), func(t *testing.T) {
+			s := multiWordReplace(c.input, c.subname, c.sub)
+			if s != c.output {
+				t.Fatalf("Expected %s got %s", c.output, s)
+			}
+		})
+	}
+}

main.go

@@ -14,7 +14,9 @@
 
 package main
 
-import "github.com/aquasecurity/kube-bench/cmd"
+import (
+	"github.com/aquasecurity/kube-bench/cmd"
+)
 
 func main() {
 	cmd.Execute()