Merge pull request #169 from mikekim/fix-1.3.7

Fixing 1.3.7 on 1.11 master.

.goreleaser.yml

@@ -13,6 +13,7 @@ nfpm:
   vendor: Aqua Security
   description: "The Kubernetes Bench for Security is a Go application that checks whether Kubernetes is deployed according to security best practices"
   license: Apache-2.0
+  homepage: https://github.com/aquasecurity/kube-bench
   formats:
     - deb
     - rpm

.travis.yml

@@ -1,6 +1,11 @@
 ---
 language: go
 
+sudo: required
+
+services:
+  - docker 
+
 notifications:
   email: false
 
@@ -10,12 +15,15 @@ before_install:
   - gem install --no-ri --no-rdoc fpm
 
 install:
-  - go get -v github.com/Masterminds/glide
-  - cd $GOPATH/src/github.com/Masterminds/glide && git checkout tags/v0.12.3 && go install && cd - # use a known good glide version
-  - glide install
+  - go get -v github.com/golang/dep/cmd/dep
+  - dep ensure -v -vendor-only
 
 script:
   - go test ./...
+  - docker build --tag kube-bench .
+  - docker run -v `pwd`:/host kube-bench install 
+  - test -d cfg
+  - test -f kube-bench
 
 after_success:
   - test -n "$TRAVIS_TAG" && curl -sL https://git.io/goreleaser | bash

Dockerfile

@@ -1,13 +1,22 @@
-FROM golang:1.9
-WORKDIR /kube-bench
-RUN go get github.com/aquasecurity/kube-bench
+FROM golang:1.9 AS build
+WORKDIR /go/src/github.com/aquasecurity/kube-bench/
+ADD Gopkg.toml Gopkg.lock ./
+RUN go get -v github.com/golang/dep/cmd/dep && dep ensure -v -vendor-only
+ADD main.go .
+ADD check/ check/
+ADD cmd/ cmd/
+RUN CGO_ENABLED=0 go install -a -ldflags '-w'
 
-FROM alpine:latest
-WORKDIR /
-COPY --from=0 /go/bin/kube-bench /kube-bench 
-COPY --from=0 /go/src/github.com/aquasecurity/kube-bench/cfg /cfg
-COPY --from=0 /go/src/github.com/aquasecurity/kube-bench/entrypoint.sh /entrypoint.sh
-ENTRYPOINT /entrypoint.sh
+FROM alpine:3.7 AS run
+WORKDIR /opt/kube-bench/
+# add GNU ps for -C, -o cmd, and --no-headers support
+# https://github.com/aquasecurity/kube-bench/issues/109
+RUN apk --no-cache add procps
+COPY --from=build /go/bin/kube-bench /usr/local/bin/kube-bench
+ADD entrypoint.sh .
+ADD cfg/ cfg/
+ENTRYPOINT ["./entrypoint.sh"]
+CMD ["install"]
 
 # Build-time metadata as defined at http://label-schema.org
 ARG BUILD_DATE

Gopkg.lock

@@ -0,0 +1,153 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  name = "github.com/fatih/color"
+  packages = ["."]
+  revision = "570b54cabe6b8eb0bc2dfce68d964677d63b5260"
+  version = "v1.5.0"
+
+[[projects]]
+  name = "github.com/fsnotify/fsnotify"
+  packages = ["."]
+  revision = "4da3e2cfbabc9f751898f250b49f2439785783a1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/glog"
+  packages = ["."]
+  revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
+
+[[projects]]
+  name = "github.com/hashicorp/hcl"
+  packages = [
+    ".",
+    "hcl/ast",
+    "hcl/parser",
+    "hcl/scanner",
+    "hcl/strconv",
+    "hcl/token",
+    "json/parser",
+    "json/scanner",
+    "json/token"
+  ]
+  revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8"
+
+[[projects]]
+  name = "github.com/inconshreveable/mousetrap"
+  packages = ["."]
+  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+  version = "v1.0"
+
+[[projects]]
+  name = "github.com/jinzhu/gorm"
+  packages = [
+    ".",
+    "dialects/postgres"
+  ]
+  revision = "5174cc5c242a728b435ea2be8a2f7f998e15429b"
+  version = "v1.0"
+
+[[projects]]
+  name = "github.com/jinzhu/inflection"
+  packages = ["."]
+  revision = "1c35d901db3da928c72a72d8458480cc9ade058f"
+
+[[projects]]
+  name = "github.com/lib/pq"
+  packages = [
+    ".",
+    "hstore",
+    "oid"
+  ]
+  revision = "83612a56d3dd153a94a629cd64925371c9adad78"
+
+[[projects]]
+  name = "github.com/magiconair/properties"
+  packages = ["."]
+  revision = "49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934"
+
+[[projects]]
+  name = "github.com/mattn/go-colorable"
+  packages = ["."]
+  revision = "5411d3eea5978e6cdc258b30de592b60df6aba96"
+
+[[projects]]
+  name = "github.com/mattn/go-isatty"
+  packages = ["."]
+  revision = "57fdcb988a5c543893cc61bce354a6e24ab70022"
+
+[[projects]]
+  name = "github.com/mitchellh/mapstructure"
+  packages = ["."]
+  revision = "06020f85339e21b2478f756a78e295255ffa4d6a"
+
+[[projects]]
+  name = "github.com/pelletier/go-toml"
+  packages = ["."]
+  revision = "0131db6d737cfbbfb678f8b7d92e55e27ce46224"
+
+[[projects]]
+  name = "github.com/spf13/afero"
+  packages = [
+    ".",
+    "mem"
+  ]
+  revision = "57afd63c68602b63ed976de00dd066ccb3c319db"
+
+[[projects]]
+  name = "github.com/spf13/cast"
+  packages = ["."]
+  revision = "acbeb36b902d72a7a4c18e8f3241075e7ab763e4"
+  version = "v1.1.0"
+
+[[projects]]
+  name = "github.com/spf13/cobra"
+  packages = ["."]
+  revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
+  version = "v0.0.1"
+
+[[projects]]
+  name = "github.com/spf13/jwalterweatherman"
+  packages = ["."]
+  revision = "12bd96e66386c1960ab0f74ced1362f66f552f7b"
+
+[[projects]]
+  name = "github.com/spf13/pflag"
+  packages = ["."]
+  revision = "4c012f6dcd9546820e378d0bdda4d8fc772cdfea"
+
+[[projects]]
+  name = "github.com/spf13/viper"
+  packages = ["."]
+  revision = "25b30aa063fc18e48662b86996252eabdcf2f0c7"
+  version = "v1.0.0"
+
+[[projects]]
+  name = "golang.org/x/sys"
+  packages = ["unix"]
+  revision = "e24f485414aeafb646f6fca458b0bf869c0880a1"
+
+[[projects]]
+  name = "golang.org/x/text"
+  packages = [
+    "internal/gen",
+    "internal/triegen",
+    "internal/ucd",
+    "transform",
+    "unicode/cldr",
+    "unicode/norm"
+  ]
+  revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
+
+[[projects]]
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  revision = "c95af922eae69f190717a0b7148960af8c55a072"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "8d9a1b665b338530deef434f168913ba1184f835aa5bfed3a213a14c613bc17e"
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -0,0 +1,23 @@
+[[constraint]]
+  name = "github.com/fatih/color"
+  version = "1.5.0"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/golang/glog"
+
+[[constraint]]
+  name = "github.com/jinzhu/gorm"
+  version = "1.0.0"
+
+[[constraint]]
+  name = "github.com/spf13/cobra"
+  version = "0.0.1"
+
+[[constraint]]
+  name = "github.com/spf13/viper"
+  version = "1.0.0"
+
+[prune]
+  go-tests = true
+  unused-packages = true

README.md

@@ -3,62 +3,91 @@
 [![Docker image](https://images.microbadger.com/badges/image/aquasec/kube-bench.svg)](https://microbadger.com/images/aquasec/kube-bench "Get your own image badge on microbadger.com")
 [![Source commit](https://images.microbadger.com/badges/commit/aquasec/kube-bench.svg)](https://microbadger.com/images/aquasec/kube-bench)
 
-# kube-bench
+<img src="images/kube-bench.png" width="200" alt="kube-bench logo">
 
-The Kubernetes Bench for Security is a Go application that checks whether Kubernetes is deployed securely by running the checks documented in the CIS Kubernetes Benchmark.
+kube-bench is a Go application that checks whether Kubernetes is deployed securely by running the checks documented in the [CIS Kubernetes Benchmark](https://www.cisecurity.org/benchmark/kubernetes/).
 
-Tests are configured with YAML files, making this tool easy to update as test specifications evolve. 
+Tests are configured with YAML files, making this tool easy to update as test specifications evolve.
 
 ![Kubernetes Bench for Security](https://raw.githubusercontent.com/aquasecurity/kube-bench/master/images/output.png "Kubernetes Bench for Security")
 
 ## CIS Kubernetes Benchmark support
 
-kube-bench supports the tests for multiple versions of Kubernetes (1.6, 1.7 and 1.8) as defined in the CIS Benchmarks 1.0.0, 1.1.0 and 1.2.0 respectively. It will determine the test set to run based on the Kubernetes version running on the machine. 
+kube-bench supports the tests for multiple versions of Kubernetes (1.6, 1.7, 1.8, and 1.11) as defined in the CIS Benchmarks 1.0.0, 1.1.0, 1.2.0, and 1.30 respectively. It will determine the test set to run based on the Kubernetes version running on the machine.
 
 ## Installation
 
-You can either install kube-bench through a dedicated container, or compile it from source:
+You can choose to
+* run kube-bench from inside a container (sharing PID namespace with the host)
+* run a container that installs kube-bench on the host, and then run kube-bench directly on the host
+* install the latest binaries from the [Releases page](https://github.com/aquasecurity/kube-bench/releases),
+* compile it from source.
 
-1. Container installation:
-Run ```docker run --rm -v `pwd`:/host aquasec/kube-bench:latest```. This will copy the kube-bench binary and configuration to you host. You can then run ```./kube-bench <master|node>```.
+### Running inside a container
+
+You can avoid installing kube-bench on the host by running it inside a container using the host PID namespace.
+
+```
+docker run --pid=host -t aquasec/kube-bench:latest <master|node>
+```
+
+You can even use your own configs by mounting them over the default ones in `/opt/kube-bench/cfg/`
+
+```
+docker run --pid=host -t -v path/to/my-config.yaml:/opt/kube-bench/cfg/config.yaml aquasec/kube-bench:latest <master|node>
+```
+
+> Note: the tests require either the kubelet or kubectl binary in the path in order to know the Kubernetes version. You can pass `-v $(which kubectl):/usr/bin/kubectl` to the above invocations to resolve this.
+
+### Running in a kubernetes cluster
+Run the master check
+
+```
+kubectl run --rm -i -t kube-bench-master --image=aquasec/kube-bench:latest --restart=Never --overrides="{ \"apiVersion\": \"v1\", \"spec\": { \"hostPID\": true, \"nodeSelector\": { \"kubernetes.io/role\": \"master\" }, \"tolerations\": [ { \"key\": \"node-role.kubernetes.io/master\", \"operator\": \"Exists\", \"effect\": \"NoSchedule\" } ] } }" -- master --version 1.11
+```
+
+Run the node check
+
+```
+kubectl run --rm -i -t kube-bench-node --image=aquasec/kube-bench:latest --restart=Never --overrides="{ \"apiVersion\": \"v1\", \"spec\": { \"hostPID\": true } }" -- node --version 1.11
+```
+
+### Installing from a container
+
+This command copies the kube-bench binary and configuration files to your host from the Docker container:
+```
+docker run --rm -v `pwd`:/host aquasec/kube-bench:latest install
+```
+
+You can then run `./kube-bench <master|node>`.
+
+### Installing from sources
 
-2. Install from sources:
 If Go is installed on the target machines, you can simply clone this repository and run as follows (assuming your [$GOPATH is set](https://github.com/golang/go/wiki/GOPATH)):
 
-```go get github.com/aquasecurity/kube-bench
-go get github.com/Masterminds/glide
+```shell
+go get github.com/aquasecurity/kube-bench
+go get github.com/golang/dep/cmd/dep
 cd $GOPATH/src/github.com/aquasecurity/kube-bench
-$GOPATH/bin/glide install
-go build -o kube-bench . 
-./kube-bench <master|node>
-```
+$GOPATH/bin/dep ensure -vendor-only
+go build -o kube-bench .
 
-## Usage
-```./kube-bench [command]```
+# See all supported options
+./kube-bench --help
 
-```
-Available Commands:
-  federated   Run benchmark checks for a Kubernetes federated deployment.
-  help        Help about any command
-  master      Run benchmark checks for a Kubernetes master node.
-  node        Run benchmark checks for a Kubernetes node.
+# Run the all checks on a master node
+./kube-bench master
 
-Flags:
-  -c, --check string          A comma-delimited list of checks to run as specified in CIS document. Example --check="1.1.1,1.1.2"
-      --config string         config file (default is ./cfg/config.yaml)
-  -g, --group string          Run all the checks under this comma-delimited list of groups. Example --group="1.1"
-      --json                  Prints the results as JSON
-  -v, --verbose               verbose output (default false)
 ```
 
 ## Configuration
-Kubernetes config and binary file locations and names can vary from installation to installation, so these are configurable in the `cfg/config.yaml` file. 
+Kubernetes config and binary file locations and names can vary from installation to installation, so these are configurable in the `cfg/config.yaml` file.
 
-For each type of node (*master*, *node* or *federated*) there is a list of components, and for each component there is a set of binaries (*bins*) and config files (*confs*) that kube-bench will look for (in the order they are listed). If your installation uses a different binary name or config file location for a Kubernetes component, you can add it to `cfg/config.yaml`.  
+For each type of node (*master*, *node* or *federated*) there is a list of components, and for each component there is a set of binaries (*bins*) and config files (*confs*) that kube-bench will look for (in the order they are listed). If your installation uses a different binary name or config file location for a Kubernetes component, you can add it to `cfg/config.yaml`.
 
-* **bins** - If there is a *bins* list for a component, at least one of these binaries must be running. The tests will consider the parameters for the first binary in the list found to be running. 
-* **podspecs** - From version 1.2.0 of the benchmark (tests for Kubernetes 1.8), the remediation instructions were updated to assume that the configuration for several kubernetes components is defined in a pod YAML file, and podspec settings define where to look for that configuration. 
-* **confs** - If one of the listed config files is found, this will be considered for the test. Tests can continue even if no config file is found. If no file is found at any of the listed locations, and a *defaultconf* location is given for the component, the test will give remediation advice using the *defaultconf* location. 
+* **bins** - If there is a *bins* list for a component, at least one of these binaries must be running. The tests will consider the parameters for the first binary in the list found to be running.
+* **podspecs** - From version 1.2.0 of the benchmark (tests for Kubernetes 1.8), the remediation instructions were updated to assume that the configuration for several kubernetes components is defined in a pod YAML file, and podspec settings define where to look for that configuration.
+* **confs** - If one of the listed config files is found, this will be considered for the test. Tests can continue even if no config file is found. If no file is found at any of the listed locations, and a *defaultconf* location is given for the component, the test will give remediation advice using the *defaultconf* location.
 * **unitfiles** - From version 1.2.0 of the benchmark  (tests for Kubernetes 1.8), the remediation instructions were updated to assume that kubelet configuration is defined in a service file, and this setting defines where to look for that configuration.
 
 ## Test config YAML representation
@@ -118,7 +147,7 @@ These operations are:
 - `has`: tests if the flag value contains the compared value.
 - `nothave`: tests if the flag value does not contain the compared value.
 
-# Roadmap 
-Going forward we plan to release updates to kube-bench to add support for new releases of the Benchmark, which in turn we can anticipate being made for each new Kubernetes release. 
+# Roadmap
+Going forward we plan to release updates to kube-bench to add support for new releases of the Benchmark, which in turn we can anticipate being made for each new Kubernetes release.
 
-We welcome PRs and issue reports. 
+We welcome PRs and issue reports.

cfg/1.11/config.yaml

@@ -0,0 +1,29 @@
+---
+## Controls Files.
+# These are YAML files that hold all the details for running checks.
+#
+## Uncomment to use different control file paths.
+# masterControls: ./cfg/master.yaml
+# nodeControls: ./cfg/node.yaml
+# federatedControls: ./cfg/federated.yaml
+
+master:
+  apiserver:
+    defaultconf: /etc/kubernetes/manifests/kube-apiserver.yaml
+
+  scheduler:
+    defaultconf: /etc/kubernetes/manifests/kube-scheduler.yaml
+
+  controllermanager:
+    defaultconf: /etc/kubernetes/manifests/kube-controller-manager.yaml
+
+  etcd:
+    defaultconf: /etc/kubernetes/manifests/etcd.yaml
+
+node:
+  kubelet:
+    defaultconf: /etc/kubernetes/kubelet.conf
+    defaultsvc: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+
+  proxy:
+    defaultconf: /etc/kubernetes/addons/kube-proxy-daemonset.yaml

cfg/1.11/node.yaml

@@ -0,0 +1,497 @@
+---
+controls:
+version: 1.11
+id: 2
+text: "Worker Node Security Configuration"
+type: "node"
+groups:
+- id: 2.1
+  text: "Kubelet"
+  checks:
+  - id: 2.1.1
+    text: "Ensure that the --allow-privileged argument is set to false (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--allow-privileged"
+        compare:
+          op: eq
+          value: false
+        set: true
+    remediation: |
+      Edit the kubelet service file $kubeletconf
+      on each worker node and set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable.
+      --allow-privileged=false
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.2
+    text: "Ensure that the --anonymous-auth argument is set to false (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--anonymous-auth"
+        compare:
+          op: eq
+          value: false
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set authentication: anonymous: enabled to
+      false .
+      If using executable arguments, edit the kubelet service file
+      $kubeletconf on each worker node and
+      set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable.
+      --anonymous-auth=false
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.3
+    text: "Ensure that the --authorization-mode argument is not set to AlwaysAllow (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--authorization-mode"
+        compare:
+          op: nothave
+          value: "AlwaysAllow"
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set authorization: mode to Webhook.
+      If using executable arguments, edit the kubelet service file
+      $kubeletconf on each worker node and
+      set the below parameter in KUBELET_AUTHZ_ARGS variable.
+      --authorization-mode=Webhook
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.4
+    text: "Ensure that the --client-ca-file argument is set as appropriate (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--client-ca-file"
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set authentication: x509: clientCAFile to
+      the location of the client CA file.
+      If using command line arguments, edit the kubelet service file
+      $kubeletconf on each worker node and
+      set the below parameter in KUBELET_AUTHZ_ARGS variable.
+      --client-ca-file=<path/to/client-ca-file>
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.5
+    text: "Ensure that the --read-only-port argument is set to 0 (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--read-only-port"
+        compare:
+          op: eq
+          value: 0
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set readOnlyPort to 0 .
+      If using command line arguments, edit the kubelet service file
+      $kubeletconf on each worker node and
+      set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable.
+      --read-only-port=0
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.6
+    text: "Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--streaming-connection-idle-timeout"
+        compare:
+          op: noteq
+          value: 0
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set streamingConnectionIdleTimeout to a
+      value other than 0.
+      If using command line arguments, edit the kubelet service file
+      $kubeletconf on each worker node and
+      set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable.
+      --streaming-connection-idle-timeout=5m
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.7
+    text: "Ensure that the --protect-kernel-defaults argument is set to true (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--protect-kernel-defaults"
+        compare:
+          op: eq
+          value: true
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set protectKernelDefaults: true .
+      If using command line arguments, edit the kubelet service file
+      $kubeletconf on each worker node and
+      set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable.
+      --protect-kernel-defaults=true
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.8
+    text: "Ensure that the --make-iptables-util-chains argument is set to true (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      bin_op: or
+      test_items:
+      - flag: "--make-iptables-util-chains"
+        compare:
+          op: eq
+          value: true
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set makeIPTablesUtilChains: true .
+      If using command line arguments, edit the kubelet service file
+      $kubeletconf on each worker node and
+      remove the --make-iptables-util-chains argument from the
+      KUBELET_SYSTEM_PODS_ARGS variable.
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.9
+    text: "Ensure that the --hostname-override argument is not set (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--hostname-override"
+        set: false
+    remediation: |
+      Edit the kubelet service file $kubeletconf
+      on each worker node and remove the --hostname-override argument from the
+      KUBELET_SYSTEM_PODS_ARGS variable.
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.10
+    text: "Ensure that the --event-qps argument is set to 0 (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--event-qps"
+        compare:
+          op: eq
+          value: 0
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set eventRecordQPS: 0 .
+      If using command line arguments, edit the kubelet service file
+      $kubeletconf on each worker node and
+      set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable.
+      --event-qps=0
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.11
+    text: "Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      bin_op: and
+      test_items:
+      - flag: "--tls-cert-file"
+        set: true
+      - flag: "--tls-private-key-file"
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set tlsCertFile to the location of the certificate
+      file to use to identify this Kubelet, and tlsPrivateKeyFile to the location of the
+      corresponding private key file.
+      If using command line arguments, edit the kubelet service file
+      $kubeletconf on each worker node and
+      set the below parameters in KUBELET_CERTIFICATE_ARGS variable.
+      --tls-cert-file=<path/to/tls-certificate-file>
+      file=<path/to/tls-key-file>
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.12
+    text: "Ensure that the --cadvisor-port argument is set to 0 (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      bin_op: or
+      test_items:
+      - flag: "--cadvisor-port"
+        compare:
+          op: eq
+          value: 0
+        set: true
+      - flag: "--cadvisor-port"
+        set: false
+    remediation: |
+      Edit the kubelet service file $kubeletconf
+      on each worker node and set the below parameter in KUBELET_CADVISOR_ARGS variable.
+      --cadvisor-port=0
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.13
+    text: "Ensure that the --rotate-certificates argument is not set to false (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--rotate-certificates"
+        compare:
+          op: eq
+          value: true
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to add the line rotateCertificates: true.
+      If using command line arguments, edit the kubelet service file $kubeletconf 
+      on each worker node and add --rotate-certificates=true argument to the KUBELET_CERTIFICATE_ARGS variable.
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.14
+    text: "Ensure that the RotateKubeletServerCertificate argument is set to true (Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "RotateKubeletServerCertificate"
+        compare:
+          op: eq
+          value: true
+        set: true
+    remediation: |
+      Edit the kubelet service file $kubeletconf
+      on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable.
+      --feature-gates=RotateKubeletServerCertificate=true
+      Based on your system, restart the kubelet service. For example:
+      systemctl daemon-reload
+      systemctl restart kubelet.service
+    scored: true
+
+  - id: 2.1.15
+    text: "Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Not Scored)"
+    audit: "ps -ef | grep $kubeletbin | grep -v grep"
+    tests:
+      test_items:
+      - flag: "--tls-cipher-suites"
+        compare:
+          op: eq
+          value: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
+        set: true
+    remediation: |
+      If using a Kubelet config file, edit the file to set TLSCipherSuites: to TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 ,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 ,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 ,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256
+      If using executable arguments, edit the kubelet service file $kubeletconf on each worker node and set the below parameter.
+      --tls-cipher- suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM _SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM _SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM _SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256
+    scored: false
+
+- id: 2.2
+  text: "Configuration Files"
+  checks:
+    - id: 2.2.1
+      text: "Ensure that the kubelet.conf file permissions are set to 644 or
+      more restrictive (Scored)"
+      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %a $kubeletconf; fi'"
+      tests:
+        bin_op: or
+        test_items:
+          - flag: "644"
+            compare:
+              op: eq
+              value: "644"
+            set: true
+          - flag: "640"
+            compare:
+              op: eq
+              value: "640"
+            set: true
+          - flag: "600"
+            compare:
+              op: eq
+              value: "600"
+            set: true
+      remediation: |
+        Run the below command (based on the file location on your system) on the each worker
+        node. For example,
+        chmod 644 $kubeletconf
+      scored: true
+
+    - id: 2.2.2
+      text: "Ensure that the kubelet.conf file ownership is set to root:root (Scored)"
+      audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %U:%G $kubeletconf; fi'"
+      tests:
+        test_items:
+          - flag: "root:root"
+            compare:
+              op: eq
+              value: root:root
+            set: true
+      remediation: |
+        Run the below command (based on the file location on your system) on the each worker
+        node. For example,
+        chown root:root $kubeletconf
+      scored: true
+
+    - id: 2.2.3
+      text: "Ensure that the kubelet service file permissions are set to 644 or
+      more restrictive (Scored)"
+      audit: "/bin/sh -c 'if test -e $kubeletsvc; then stat -c %a $kubeletsvc; fi'"
+      tests:
+        bin_op: or
+        test_items:
+        - flag: "644"
+          compare:
+            op: eq
+            value: 644
+          set: true
+        - flag: "640"
+          compare:
+            op: eq
+            value: "640"
+          set: true
+        - flag: "600"
+          compare:
+            op: eq
+            value: "600"
+          set: true
+      remediation: |
+        Run the below command (based on the file location on your system) on the each worker
+        node. For example,
+        chmod 755 $kubeletsvc
+      scored: true
+
+    - id: 2.2.4
+      text: "Ensure that the kubelet service file ownership is set to root:root (Scored)"
+      audit: "/bin/sh -c 'if test -e $kubeletsvc; then stat -c %U:%G $kubeletsvc; fi'"
+      tests:
+        test_items:
+        - flag: "root:root"
+          set: true
+      remediation: |
+        Run the below command (based on the file location on your system) on the each worker
+        node. For example,
+        chown root:root $kubeletsvc
+      scored: true
+
+    - id: 2.2.5
+      text: "Ensure that the proxy kubeconfig file permissions are set to 644 or more restrictive (Scored)"
+      audit: "/bin/sh -c 'if test -e $proxyconf; then stat -c %a $proxyconf; fi'"
+      tests:
+        bin_op: or
+        test_items:
+        - flag: "644"
+          compare:
+            op: eq
+            value: "644"
+          set: true
+        - flag: "640"
+          compare:
+            op: eq
+            value: "640"
+          set: true
+        - flag: "600"
+          compare:
+            op: eq
+            value: "600"
+          set: true
+      remediation: |
+        Run the below command (based on the file location on your system) on the each worker
+        node. For example,
+        chmod 644 $proxyconf
+      scored: true
+
+    - id: 2.2.6
+      text: "Ensure that the proxy kubeconfig file ownership is set to root:root (Scored)"
+      audit: "/bin/sh -c 'if test -e $proxyconf; then stat -c %U:%G $proxyconf; fi'"
+      tests:
+        test_items:
+        - flag: "root:root"
+          set: true
+      remediation: |
+          Run the below command (based on the file location on your system) on the each worker
+          node. For example,
+          chown root:root $proxyconf
+      scored: true
+
+    - id: 2.2.7
+      text: "Ensure that the certificate authorities file permissions are set to
+      644 or more restrictive (Scored)"
+      type: manual
+      remediation: |
+        Run the following command to modify the file permissions of the --client-ca-file
+        chmod 644 <filename>
+      scored: true
+
+    - id: 2.2.8
+      text: "Ensure that the client certificate authorities file ownership is set to root:root (Scored)"
+      audit: "/bin/sh -c 'if test -e $ca-file; then stat -c %U:%G $ca-file; fi'"
+      type: manual
+      remediation: |
+        Run the following command to modify the ownership of the --client-ca-file .
+        chown root:root <filename>
+      scored: true
+
+    - id: 2.2.9
+      text: "Ensure that the kubelet configuration file ownership is set to root:root (Scored)"
+      audit: "/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then stat -c %U:%G /var/lib/kubelet/config.yaml; fi'"
+      tests:
+        test_items:
+        - flag: "root:root"
+          set: true
+      remediation: |
+        Run the following command (using the config file location identied in the Audit step)
+        chown root:root /etc/kubernetes/kubelet.conf
+      scored: true
+
+    - id: 2.2.10
+      text: "Ensure that the kubelet configuration file has permissions set to 644 or more restrictive (Scored)"
+      audit: "/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then stat -c %a /var/lib/kubelet/config.yaml; fi'"
+      tests:
+        bin_op: or
+        test_items:
+        - flag: "644"
+          compare:
+            op: eq
+            value: "644"
+          set: true
+        - flag: "640"
+          compare:
+            op: eq
+            value: "640"
+          set: true
+        - flag: "600"
+          compare:
+            op: eq
+            value: "600"
+          set: true
+      remediation: |
+        Run the following command (using the config file location identied in the Audit step)
+        chmod 644 /var/lib/kubelet/config.yaml
+      scored: true

cfg/1.8/config.yaml

@@ -9,21 +9,27 @@
 
 master:
   apiserver:
+    confs:
+      - /etc/kubernetes/manifests/kube-apiserver.yaml
+      - /etc/kubernetes/manifests/kube-apiserver.manifest
     defaultconf: /etc/kubernetes/manifests/kube-apiserver.yaml
 
   scheduler:
     confs: 
       - /etc/kubernetes/manifests/kube-scheduler.yaml
+      - /etc/kubernetes/manifests/kube-scheduler.manifest
     defaultconf: /etc/kubernetes/manifests/kube-scheduler.yaml
 
   controllermanager:
     confs:
       - /etc/kubernetes/manifests/kube-controller-manager.yaml
+      - /etc/kubernetes/manifests/kube-controller-manager.manifest
     defaultconf: /etc/kubernetes/manifests/kube-controller-manager.yaml
 
   etcd:
     confs:
       - /etc/kubernetes/manifests/etcd.yaml
+      - /etc/kubernetes/manifests/etcd.manifest
     defaultconf: /etc/kubernetes/manifests/etcd.yaml
 
 node:

cfg/1.8/master.yaml

@@ -418,7 +418,7 @@ groups:
 
   - id: 1.1.26
     text: "Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as 
-      appropriate (Scored"
+      appropriate (Scored)"
     audit: "ps -ef | grep $apiserverbin | grep -v grep"
     tests:
       bin_op: and
@@ -610,7 +610,7 @@ groups:
     remediation: |
       Edit the API server pod specification file $apiserverconf
       and set the below parameter as appropriate and if needed. For example,
-      --request-timeout=300
+      --request-timeout=300s
     scored: true
 
 - id: 1.2
@@ -666,7 +666,7 @@ groups:
     scored: true
 
   - id: 1.3.3
-    text: "Ensure that the --use-service-account-credentials argument is set"
+    text: "Ensure that the --use-service-account-credentials argument is set (Scored)"
     audit: "ps -ef | grep $controllermanagerbin | grep -v grep"
     tests:
       test_items:
@@ -942,7 +942,7 @@ groups:
 
   - id: 1.4.11
     text: "Ensure that the etcd data directory permissions are set to 700 or more restrictive (Scored)"
-    audit: ps -ef | grep $etcdbin | grep -v grep | sed 's%.*data-dir[= ]\([^ ]*\).*%\1%' | xargs stat -c %a
+    audit: ps -ef | grep $etcdbin | grep -- --data-dir | sed 's%.*data-dir[= ]\([^ ]*\).*%\1%' | xargs stat -c %a
     tests:
       test_items:
       - flag: "700"
@@ -960,7 +960,7 @@ groups:
 
   - id: 1.4.12
     text: "Ensure that the etcd data directory ownership is set to etcd:etcd (Scored)"
-    audit: ps -ef | grep $etcdbin | grep -v grep | sed 's%.*data-dir[= ]\(\S*\)%\1%' | xargs stat -c %U:%G
+    audit: ps -ef | grep $etcdbin | grep -- --data-dir | sed 's%.*data-dir[= ]\([^ ]*\).*%\1%' | xargs stat -c %U:%G
     tests:
       test_items:
       - flag: "etcd:etcd"

cfg/1.8/node.yaml

@@ -368,8 +368,7 @@ groups:
       scored: true
 
     - id: 2.2.4
-      text: "Ensure that the kubelet service file permissions are set to 644 or
-      more restrictive (Scored)"
+      text: "2.2.4 Ensure that the kubelet service file ownership is set to root:root (Scored)"
       audit: "/bin/sh -c 'if test -e $kubeletconf; then stat -c %U:%G $kubeletconf; fi'"
       tests:
         test_items:
@@ -411,6 +410,7 @@ groups:
 
     - id: 2.2.6
       text: "Ensure that the proxy kubeconfig file ownership is set to root:root (Scored)"
+      audit: "/bin/sh -c 'if test -e $proxyconf; then stat -c %U:%G $proxyconf; fi'"
       tests:
         test_items:
         - flag: "root:root"

check/check.go

@@ -60,16 +60,17 @@ func handleError(err error, context string) (errmsg string) {
 // Check contains information about a recommendation in the
 // CIS Kubernetes 1.6+ document.
 type Check struct {
-	ID          string 		`yaml:"id" json:"test_number"`
-	Text        string		`json:"test_desc"`
+	ID          string      `yaml:"id" json:"test_number"`
+	Text        string      `json:"test_desc"`
 	Audit       string      `json:"omit"`
 	Type        string      `json:"type"`
 	Commands    []*exec.Cmd `json:"omit"`
 	Tests       *tests      `json:"omit"`
 	Set         bool        `json:"omit"`
-	Remediation string 		`json:"-"`
-	TestInfo    []string 	`json:"test_info"`
-	State 					`json:"status"`
+	Remediation string      `json:"-"`
+	TestInfo    []string    `json:"test_info"`
+	State       `json:"status"`
+	ActualValue string `json:"actual_value"`
 }
 
 // Run executes the audit commands specified in a check and outputs
@@ -157,15 +158,25 @@ func (c *Check) Run() {
 		i++
 	}
 
-	if errmsgs != "" {
-		glog.V(2).Info(errmsgs)
+	finalOutput := c.Tests.execute(out.String())
+	if finalOutput != nil {
+		c.ActualValue = finalOutput.actualResult
+		if finalOutput.testResult {
+			c.State = PASS
+		} else {
+			c.State = FAIL
+		}
+	} else {
+		errmsgs += handleError(
+			fmt.Errorf("final output is nil"),
+			fmt.Sprintf("failed to run: %s\n",
+				c.Audit,
+			),
+		)
 	}
 
-	res := c.Tests.execute(out.String())
-	if res {
-		c.State = PASS
-	} else {
-		c.State = FAIL
+	if errmsgs != "" {
+		glog.V(2).Info(errmsgs)
 	}
 }
 

check/controls.go

@@ -23,9 +23,9 @@ import (
 
 // Controls holds all controls to check for master nodes.
 type Controls struct {
-	ID      string `yaml:"id" json:"id"`
-	Version string	`json:"version"`
-	Text    string 	`json:"text"`
+	ID      string   `yaml:"id" json:"id"`
+	Version string   `json:"version"`
+	Text    string   `json:"text"`
 	Type    NodeType `json:"node_type"`
 	Groups  []*Group `json:"tests"`
 	Summary
@@ -43,9 +43,9 @@ type Group struct {
 
 // Summary is a summary of the results of control checks run.
 type Summary struct {
-	Pass int   `json:"total_pass"`
-	Fail int   `json:"total_fail"`
-	Warn int   `json:"total_warn"`
+	Pass int `json:"total_pass"`
+	Fail int `json:"total_fail"`
+	Warn int `json:"total_warn"`
 }
 
 // NewControls instantiates a new master Controls object.

check/data

@@ -17,7 +17,7 @@ groups:
     - id: 1
       text: "flag is not set"
       tests:
-        test_item:
+        test_items:
           - flag: "--basic-auth"
             set: false
 

check/test.go

@@ -49,8 +49,13 @@ type compare struct {
 	Value string
 }
 
-func (t *testItem) execute(s string) (result bool) {
-	result = false
+type testOutput struct {
+	testResult   bool
+	actualResult string
+}
+
+func (t *testItem) execute(s string) *testOutput {
+	result := &testOutput{}
 	match := strings.Contains(s, t.Flag)
 
 	if t.Set {
@@ -78,45 +83,57 @@ func (t *testItem) execute(s string) (result bool) {
 				os.Exit(1)
 			}
 
+			result.actualResult = strings.ToLower(flagVal)
 			switch t.Compare.Op {
 			case "eq":
-				result = flagVal == t.Compare.Value
+				value := strings.ToLower(flagVal)
+				// Do case insensitive comparaison for booleans ...
+				if value == "false" || value == "true" {
+					result.testResult = value == t.Compare.Value
+				} else {
+					result.testResult = flagVal == t.Compare.Value
+				}
 
 			case "noteq":
-				result = !(flagVal == t.Compare.Value)
+				value := strings.ToLower(flagVal)
+				// Do case insensitive comparaison for booleans ...
+				if value == "false" || value == "true" {
+					result.testResult = !(value == t.Compare.Value)
+				} else {
+					result.testResult = !(flagVal == t.Compare.Value)
+				}
 
 			case "gt":
 				a, b := toNumeric(flagVal, t.Compare.Value)
-				result = a > b
+				result.testResult = a > b
 
 			case "gte":
 				a, b := toNumeric(flagVal, t.Compare.Value)
-				result = a >= b
+				result.testResult = a >= b
 
 			case "lt":
 				a, b := toNumeric(flagVal, t.Compare.Value)
-				result = a < b
+				result.testResult = a < b
 
 			case "lte":
 				a, b := toNumeric(flagVal, t.Compare.Value)
-				result = a <= b
+				result.testResult = a <= b
 
 			case "has":
-				result = strings.Contains(flagVal, t.Compare.Value)
+				result.testResult = strings.Contains(flagVal, t.Compare.Value)
 
 			case "nothave":
-				result = !strings.Contains(flagVal, t.Compare.Value)
+				result.testResult = !strings.Contains(flagVal, t.Compare.Value)
 			}
 		} else {
-			result = isset
+			result.testResult = isset
 		}
 
 	} else {
 		notset := !match
-		result = notset
+		result.testResult = notset
 	}
-
-	return
+	return result
 }
 
 type tests struct {
@@ -124,13 +141,19 @@ type tests struct {
 	BinOp     binOp       `yaml:"bin_op"`
 }
 
-func (ts *tests) execute(s string) (result bool) {
-	res := make([]bool, len(ts.TestItems))
+func (ts *tests) execute(s string) *testOutput {
+	finalOutput := &testOutput{}
 
-	for i, t := range ts.TestItems {
-		res[i] = t.execute(s)
+	res := make([]testOutput, len(ts.TestItems))
+	if len(res) == 0 {
+		return finalOutput
 	}
 
+	for i, t := range ts.TestItems {
+		res[i] = *(t.execute(s))
+	}
+
+	var result bool
 	// If no binary operation is specified, default to AND
 	switch ts.BinOp {
 	default:
@@ -139,16 +162,19 @@ func (ts *tests) execute(s string) (result bool) {
 	case and, "":
 		result = true
 		for i := range res {
-			result = result && res[i]
+			result = result && res[i].testResult
 		}
 	case or:
 		result = false
 		for i := range res {
-			result = result || res[i]
+			result = result || res[i].testResult
 		}
 	}
 
-	return
+	finalOutput.testResult = result
+	finalOutput.actualResult = res[0].actualResult
+
+	return finalOutput
 }
 
 func toNumeric(a, b string) (c, d int) {

check/test_test.go

@@ -113,7 +113,7 @@ func TestTestExecute(t *testing.T) {
 	}
 
 	for _, c := range cases {
-		res := c.Tests.execute(c.str)
+		res := c.Tests.execute(c.str).testResult
 		if !res {
 			t.Errorf("%s, expected:%v, got:%v\n", c.Text, true, res)
 		}

cmd/common.go

@@ -17,6 +17,7 @@ package cmd
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/aquasecurity/kube-bench/check"
@@ -28,64 +29,67 @@ var (
 	errmsgs string
 )
 
-func runChecks(t check.NodeType) {
+func runChecks(nodetype check.NodeType) {
 	var summary check.Summary
-	var nodetype string
 	var file string
 	var err error
 	var typeConf *viper.Viper
 
-	switch t {
+	switch nodetype {
 	case check.MASTER:
 		file = masterFile
-		nodetype = "master"
 	case check.NODE:
 		file = nodeFile
-		nodetype = "node"
 	case check.FEDERATED:
 		file = federatedFile
-		nodetype = "federated"
 	}
 
-	ver := getKubeVersion()
-	switch ver {
-	case "1.9", "1.10":
-		continueWithError(nil, fmt.Sprintf("No CIS spec for %s - using tests from CIS 1.2.0 spec for Kubernetes 1.8\n", ver))
-		ver = "1.8"
+	runningVersion, err := getKubeVersion()
+	if err != nil && kubeVersion == "" {
+		exitWithError(fmt.Errorf("Version check failed: %s\nAlternatively, you can specify the version with --version", err))
+	}
+	path, err := getConfigFilePath(kubeVersion, runningVersion, file)
+	if err != nil {
+		exitWithError(fmt.Errorf("can't find %s controls file in %s: %v", nodetype, cfgDir, err))
 	}
 
-	path := filepath.Join(cfgDir, ver)
 	def := filepath.Join(path, file)
-
 	in, err := ioutil.ReadFile(def)
 	if err != nil {
-		exitWithError(fmt.Errorf("error opening %s controls file: %v", t, err))
+		exitWithError(fmt.Errorf("error opening %s controls file: %v", nodetype, err))
 	}
 
+	glog.V(1).Info(fmt.Sprintf("Using benchmark file: %s\n", def))
+
 	// Merge kubernetes version specific config if any.
 	viper.SetConfigFile(path + "/config.yaml")
 	err = viper.MergeInConfig()
 	if err != nil {
-		continueWithError(err, fmt.Sprintf("Reading %s specific configuration file", ver))
+		if os.IsNotExist(err) {
+			glog.V(2).Info(fmt.Sprintf("No version-specific config.yaml file in %s", path))
+		} else {
+			exitWithError(fmt.Errorf("couldn't read config file %s: %v", path+"/config.yaml", err))
+		}
+	} else {
+		glog.V(1).Info(fmt.Sprintf("Using config file: %s\n", viper.ConfigFileUsed()))
 	}
-	typeConf = viper.Sub(nodetype)
 
 	// Get the set of exectuables and config files we care about on this type of node. This also
 	// checks that the executables we need for the node type are running.
+	typeConf = viper.Sub(string(nodetype))
 	binmap := getBinaries(typeConf)
 	confmap := getConfigFiles(typeConf)
+	svcmap := getServiceFiles(typeConf)
 
 	// Variable substitutions. Replace all occurrences of variables in controls files.
 	s := string(in)
 	s = makeSubstitutions(s, "bin", binmap)
 	s = makeSubstitutions(s, "conf", confmap)
+	s = makeSubstitutions(s, "svc", svcmap)
 
-	glog.V(1).Info(fmt.Sprintf("Using config file: %s\n", viper.ConfigFileUsed()))
-	glog.V(1).Info(fmt.Sprintf("Using benchmark file: %s\n", def))
-
-	controls, err := check.NewControls(t, []byte(s))
+	controls, err := check.NewControls(nodetype, []byte(s))
 	if err != nil {
-		exitWithError(fmt.Errorf("error setting up %s controls: %v", t, err))
+		exitWithError(fmt.Errorf("error setting up %s controls: %v", nodetype, err))
 	}
 
 	if groupList != "" && checkList == "" {
@@ -131,41 +135,48 @@ func colorPrint(state check.State, s string) {
 
 // prettyPrint outputs the results to stdout in human-readable format
 func prettyPrint(r *check.Controls, summary check.Summary) {
-	colorPrint(check.INFO, fmt.Sprintf("%s %s\n", r.ID, r.Text))
-	for _, g := range r.Groups {
-		colorPrint(check.INFO, fmt.Sprintf("%s %s\n", g.ID, g.Text))
-		for _, c := range g.Checks {
-			colorPrint(c.State, fmt.Sprintf("%s %s\n", c.ID, c.Text))
-		}
-	}
-
-	fmt.Println()
-
-	// Print remediations.
-	if summary.Fail > 0 || summary.Warn > 0 {
-		colors[check.WARN].Printf("== Remediations ==\n")
+	// Print check results.
+	if !noResults {
+		colorPrint(check.INFO, fmt.Sprintf("%s %s\n", r.ID, r.Text))
 		for _, g := range r.Groups {
+			colorPrint(check.INFO, fmt.Sprintf("%s %s\n", g.ID, g.Text))
 			for _, c := range g.Checks {
-				if c.State != check.PASS {
-					fmt.Printf("%s %s\n", c.ID, c.Remediation)
-				}
+				colorPrint(c.State, fmt.Sprintf("%s %s\n", c.ID, c.Text))
 			}
 		}
+
 		fmt.Println()
 	}
 
-	// Print summary setting output color to highest severity.
-	var res check.State
-	if summary.Fail > 0 {
-		res = check.FAIL
-	} else if summary.Warn > 0 {
-		res = check.WARN
-	} else {
-		res = check.PASS
+	// Print remediations.
+	if !noRemediations {
+		if summary.Fail > 0 || summary.Warn > 0 {
+			colors[check.WARN].Printf("== Remediations ==\n")
+			for _, g := range r.Groups {
+				for _, c := range g.Checks {
+					if c.State != check.PASS {
+						fmt.Printf("%s %s\n", c.ID, c.Remediation)
+					}
+				}
+			}
+			fmt.Println()
+		}
 	}
 
-	colors[res].Printf("== Summary ==\n")
-	fmt.Printf("%d checks PASS\n%d checks FAIL\n%d checks WARN\n",
-		summary.Pass, summary.Fail, summary.Warn,
-	)
+	// Print summary setting output color to highest severity.
+	if !noSummary {
+		var res check.State
+		if summary.Fail > 0 {
+			res = check.FAIL
+		} else if summary.Warn > 0 {
+			res = check.WARN
+		} else {
+			res = check.PASS
+		}
+
+		colors[res].Printf("== Summary ==\n")
+		fmt.Printf("%d checks PASS\n%d checks FAIL\n%d checks WARN\n",
+			summary.Pass, summary.Fail, summary.Warn,
+		)
+	}
 }

cmd/root.go

@@ -26,9 +26,10 @@ import (
 
 var (
 	envVarsPrefix      = "KUBE_BENCH"
-	cfgDir             = "./cfg"
 	defaultKubeVersion = "1.6"
+	kubeVersion        string
 	cfgFile            string
+	cfgDir             string
 	jsonFmt            bool
 	pgSQL              bool
 	checkList          string
@@ -36,13 +37,16 @@ var (
 	masterFile         string
 	nodeFile           string
 	federatedFile      string
+	noResults          bool
+	noSummary          bool
+	noRemediations     bool
 )
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:   os.Args[0],
 	Short: "Run CIS Benchmarks checks against a Kubernetes deployment",
-	Long:  `This tool runs the CIS Kubernetes 1.6 Benchmark v1.0.0 checks.`,
+	Long:  `This tool runs the CIS Kubernetes Benchmark (https://www.cisecurity.org/benchmark/kubernetes/)`,
 }
 
 // Execute adds all child commands to the root command sets flags appropriately.
@@ -60,8 +64,13 @@ func Execute() {
 func init() {
 	cobra.OnInitialize(initConfig)
 
+	// Output control
+	RootCmd.PersistentFlags().BoolVar(&noResults, "noresults", false, "Disable printing of results section")
+	RootCmd.PersistentFlags().BoolVar(&noSummary, "nosummary", false, "Disable printing of summary section")
+	RootCmd.PersistentFlags().BoolVar(&noRemediations, "noremediations", false, "Disable printing of remediations section")
 	RootCmd.PersistentFlags().BoolVar(&jsonFmt, "json", false, "Prints the results as JSON")
 	RootCmd.PersistentFlags().BoolVar(&pgSQL, "pgsql", false, "Save the results to PostgreSQL")
+
 	RootCmd.PersistentFlags().StringVarP(
 		&checkList,
 		"check",
@@ -77,6 +86,8 @@ func init() {
 		`Run all the checks under this comma-delimited list of groups. Example --group="1.1"`,
 	)
 	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is ./cfg/config.yaml)")
+	RootCmd.PersistentFlags().StringVarP(&cfgDir, "config-dir", "D", "./cfg/", "config directory")
+	RootCmd.PersistentFlags().StringVar(&kubeVersion, "version", "", "Manually specify Kubernetes version, automatically detected if unset")
 
 	goflag.CommandLine.VisitAll(func(goflag *goflag.Flag) {
 		RootCmd.PersistentFlags().AddGoFlag(goflag)

cmd/util.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/aquasecurity/kube-bench/check"
@@ -116,9 +118,60 @@ func getBinaries(v *viper.Viper) map[string]string {
 	return binmap
 }
 
+// getConfigFilePath locates the config files we should be using based on either the specified
+// version, or the running version of kubernetes if not specified
+func getConfigFilePath(specifiedVersion string, runningVersion string, filename string) (path string, err error) {
+	var fileVersion string
+
+	if specifiedVersion != "" {
+		fileVersion = specifiedVersion
+	} else {
+		fileVersion = runningVersion
+	}
+
+	glog.V(2).Info(fmt.Sprintf("Looking for config for version %s", fileVersion))
+
+	for {
+		path = filepath.Join(cfgDir, fileVersion)
+		file := filepath.Join(path, string(filename))
+		glog.V(2).Info(fmt.Sprintf("Looking for config file: %s\n", file))
+
+		if _, err = os.Stat(file); !os.IsNotExist(err) {
+			if specifiedVersion == "" && fileVersion != runningVersion {
+				glog.V(1).Info(fmt.Sprintf("No test file found for %s - using tests for Kubernetes %s\n", runningVersion, fileVersion))
+			}
+			return path, nil
+		}
+
+		// If we were given an explicit version to look for, don't look for any others
+		if specifiedVersion != "" {
+			return "", err
+		}
+
+		fileVersion = decrementVersion(fileVersion)
+		if fileVersion == "" {
+			return "", fmt.Errorf("no test files found <= runningVersion")
+		}
+	}
+}
+
+// decrementVersion decrements the version number
+// We want to decrement individually even through versions where we don't supply test files
+// just in case someone wants to specify their own test files for that version
+func decrementVersion(version string) string {
+	split := strings.Split(version, ".")
+	minor, err := strconv.Atoi(split[1])
+	if err != nil {
+		return ""
+	}
+	if minor <= 1 {
+		return ""
+	}
+	split[1] = strconv.Itoa(minor - 1)
+	return strings.Join(split, ".")
+}
+
 // getConfigFiles finds which of the set of candidate config files exist
-// accepts a string 't' which indicates the type of config file, conf,
-// podspec or untifile.
 func getConfigFiles(v *viper.Viper) map[string]string {
 	confmap := make(map[string]string)
 
@@ -149,6 +202,37 @@ func getConfigFiles(v *viper.Viper) map[string]string {
 	return confmap
 }
 
+// getServiceFiles finds which of the set of candidate service files exist
+func getServiceFiles(v *viper.Viper) map[string]string {
+	svcmap := make(map[string]string)
+
+	for _, component := range v.GetStringSlice("components") {
+		s := v.Sub(component)
+		if s == nil {
+			continue
+		}
+
+		// See if any of the candidate config files exist
+		svc := findConfigFile(s.GetStringSlice("svc"))
+		if svc == "" {
+			if s.IsSet("defaultsvc") {
+				svc = s.GetString("defaultsvc")
+				glog.V(2).Info(fmt.Sprintf("Using default service file name '%s' for component %s", svc, component))
+			} else {
+				// Default the service file name that we'll substitute to the name of the component
+				glog.V(2).Info(fmt.Sprintf("Missing service file for %s", component))
+				svc = component
+			}
+		} else {
+			glog.V(2).Info(fmt.Sprintf("Component %s uses service file '%s'", component, svc))
+		}
+
+		svcmap[component] = svc
+	}
+
+	return svcmap
+}
+
 // verifyBin checks that the binary specified is running
 func verifyBin(bin string) bool {
 
@@ -212,19 +296,19 @@ func multiWordReplace(s string, subname string, sub string) string {
 	return strings.Replace(s, subname, sub, -1)
 }
 
-func getKubeVersion() string {
+func getKubeVersion() (string, error) {
 	// These executables might not be on the user's path.
 	_, err := exec.LookPath("kubectl")
 
 	if err != nil {
 		_, err = exec.LookPath("kubelet")
 		if err != nil {
-			exitWithError(fmt.Errorf("Version check failed: need kubectl or kubelet binaries to get kubernetes version"))
+			return "", fmt.Errorf("need kubectl or kubelet binaries to get kubernetes version")
 		}
-		return getKubeVersionFromKubelet()
+		return getKubeVersionFromKubelet(), nil
 	}
 
-	return getKubeVersionFromKubectl()
+	return getKubeVersionFromKubectl(), nil
 }
 
 func getKubeVersionFromKubectl() string {
@@ -240,7 +324,7 @@ func getKubeVersionFromKubectl() string {
 func getKubeVersionFromKubelet() string {
 	cmd := exec.Command("kubelet", "--version")
 	out, err := cmd.CombinedOutput()
-	
+
 	if err != nil {
 		continueWithError(fmt.Errorf("%s", out), "")
 	}
@@ -275,7 +359,7 @@ func makeSubstitutions(s string, ext string, m map[string]string) string {
 			glog.V(2).Info(fmt.Sprintf("No subsitution for '%s'\n", subst))
 			continue
 		}
-		glog.V(1).Info(fmt.Sprintf("Substituting %s with '%s'\n", subst, v))
+		glog.V(2).Info(fmt.Sprintf("Substituting %s with '%s'\n", subst, v))
 		s = multiWordReplace(s, subst, v)
 	}
 

cmd/util_test.go

@@ -15,7 +15,9 @@
 package cmd
 
 import (
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"testing"
@@ -287,6 +289,81 @@ func TestGetConfigFiles(t *testing.T) {
 	}
 }
 
+func TestGetServiceFiles(t *testing.T) {
+	cases := []struct {
+		config      map[string]interface{}
+		exp         map[string]string
+		statResults []error
+	}{
+		{
+			config: map[string]interface{}{
+				"components": []string{"kubelet"},
+				"kubelet":    map[string]interface{}{"svc": []string{"kubelet", "10-kubeadm.conf"}},
+			},
+			statResults: []error{os.ErrNotExist, nil},
+			exp:         map[string]string{"kubelet": "10-kubeadm.conf"},
+		},
+		{
+			// Component "thing" isn't included in the list of components
+			config: map[string]interface{}{
+				"components": []string{"kubelet"},
+				"kubelet":    map[string]interface{}{"svc": []string{"kubelet", "10-kubeadm.conf"}},
+				"thing":      map[string]interface{}{"svc": []string{"/my/file/thing"}},
+			},
+			statResults: []error{os.ErrNotExist, nil},
+			exp:         map[string]string{"kubelet": "10-kubeadm.conf"},
+		},
+		{
+			// More than one component
+			config: map[string]interface{}{
+				"components": []string{"kubelet", "thing"},
+				"kubelet":    map[string]interface{}{"svc": []string{"kubelet", "10-kubeadm.conf"}},
+				"thing":      map[string]interface{}{"svc": []string{"/my/file/thing"}},
+			},
+			statResults: []error{os.ErrNotExist, nil, nil},
+			exp:         map[string]string{"kubelet": "10-kubeadm.conf", "thing": "/my/file/thing"},
+		},
+		{
+			// Default thing to specified default service
+			config: map[string]interface{}{
+				"components": []string{"kubelet", "thing"},
+				"kubelet":    map[string]interface{}{"svc": []string{"kubelet", "10-kubeadm.conf"}},
+				"thing":      map[string]interface{}{"svc": []string{"/my/file/thing"}, "defaultsvc": "another/thing"},
+			},
+			statResults: []error{os.ErrNotExist, nil, os.ErrNotExist},
+			exp:         map[string]string{"kubelet": "10-kubeadm.conf", "thing": "another/thing"},
+		},
+		{
+			// Default thing to component name
+			config: map[string]interface{}{
+				"components": []string{"kubelet", "thing"},
+				"kubelet":    map[string]interface{}{"svc": []string{"kubelet", "10-kubeadm.conf"}},
+				"thing":      map[string]interface{}{"svc": []string{"/my/file/thing"}},
+			},
+			statResults: []error{os.ErrNotExist, nil, os.ErrNotExist},
+			exp:         map[string]string{"kubelet": "10-kubeadm.conf", "thing": "thing"},
+		},
+	}
+
+	v := viper.New()
+	statFunc = fakestat
+
+	for id, c := range cases {
+		t.Run(strconv.Itoa(id), func(t *testing.T) {
+			for k, val := range c.config {
+				v.Set(k, val)
+			}
+			e = c.statResults
+			eIndex = 0
+
+			m := getServiceFiles(v)
+			if !reflect.DeepEqual(m, c.exp) {
+				t.Fatalf("Got %v\nExpected %v", m, c.exp)
+			}
+		})
+	}
+}
+
 func TestMakeSubsitutions(t *testing.T) {
 	cases := []struct {
 		input string
@@ -306,3 +383,45 @@ func TestMakeSubsitutions(t *testing.T) {
 		})
 	}
 }
+
+func TestGetConfigFilePath(t *testing.T) {
+	var err error
+	cfgDir, err = ioutil.TempDir("", "kube-bench-test")
+	if err != nil {
+		t.Fatalf("Failed to create temp directory")
+	}
+	defer os.RemoveAll(cfgDir)
+	d := filepath.Join(cfgDir, "1.8")
+	err = os.Mkdir(d, 0666)
+	if err != nil {
+		t.Fatalf("Failed to create temp file")
+	}
+	ioutil.WriteFile(filepath.Join(d, "master.yaml"), []byte("hello world"), 0666)
+
+	cases := []struct {
+		specifiedVersion string
+		runningVersion   string
+		succeed          bool
+		exp              string
+	}{
+		{runningVersion: "1.8", succeed: true, exp: d},
+		{runningVersion: "1.9", succeed: true, exp: d},
+		{runningVersion: "1.10", succeed: true, exp: d},
+		{runningVersion: "1.1", succeed: false},
+		{specifiedVersion: "1.8", succeed: true, exp: d},
+		{specifiedVersion: "1.9", succeed: false},
+		{specifiedVersion: "1.10", succeed: false},
+	}
+
+	for _, c := range cases {
+		t.Run(c.specifiedVersion+"-"+c.runningVersion, func(t *testing.T) {
+			path, err := getConfigFilePath(c.specifiedVersion, c.runningVersion, "/master.yaml")
+			if err != nil && c.succeed {
+				t.Fatalf("Error %v", err)
+			}
+			if path != c.exp {
+				t.Fatalf("Got %s expected %s", path, c.exp)
+			}
+		})
+	}
+}

entrypoint.sh

@@ -1,14 +1,19 @@
-#!/bin/sh
-if [ -d /host ]; then
-  mkdir -p /host/cfg/
-  yes | cp -rf /cfg/* /host/cfg/
-  yes | cp -rf /kube-bench /host/
-  echo "==============================================="
-  echo "kube-bench is now installed on your host       "
-  echo "Run ./kube-bench to perform a security check   "
-  echo "==============================================="
+#!/bin/sh -e
+if [ "$1" == "install" ]; then
+  if [ -d /host ]; then
+    mkdir -p /host/cfg/
+    yes | cp -rf cfg/* /host/cfg/
+    yes | cp -rf /usr/local/bin/kube-bench /host/
+    echo "==============================================="
+    echo "kube-bench is now installed on your host       "
+    echo "Run ./kube-bench to perform a security check   "
+    echo "==============================================="
+  else
+    echo "Usage:"
+    echo "  install: docker run --rm -v \`pwd\`:/host aquasec/kube-bench install"
+    echo "  run:     docker run --rm --pid=host aquasec/kube-bench [command]"
+    exit
+  fi
 else
-  echo "Usage:"
-  echo "  docker run --rm -v \`pwd\`:/host aquasec/kube-bench"
-  exit 
+  exec kube-bench "$@"
 fi

glide.lock

@@ -1,72 +0,0 @@
-hash: f3cf12cf95d66d315c4aef2f3d0940770bd26267f84703e53c4928b786a91c14
-updated: 2018-01-09T12:49:41.3014329-08:00
-imports:
-- name: github.com/fatih/color
-  version: 570b54cabe6b8eb0bc2dfce68d964677d63b5260
-- name: github.com/fsnotify/fsnotify
-  version: 4da3e2cfbabc9f751898f250b49f2439785783a1
-- name: github.com/golang/glog
-  version: 23def4e6c14b4da8ac2ed8007337bc5eb5007998
-- name: github.com/hashicorp/hcl
-  version: 23c074d0eceb2b8a5bfdbb271ab780cde70f05a8
-  subpackages:
-  - hcl/ast
-  - hcl/parser
-  - hcl/scanner
-  - hcl/strconv
-  - hcl/token
-  - json/parser
-  - json/scanner
-  - json/token
-- name: github.com/inconshreveable/mousetrap
-  version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
-- name: github.com/jinzhu/gorm
-  version: 5174cc5c242a728b435ea2be8a2f7f998e15429b
-  subpackages:
-  - dialects/postgres
-- name: github.com/jinzhu/inflection
-  version: 1c35d901db3da928c72a72d8458480cc9ade058f
-- name: github.com/lib/pq
-  version: 83612a56d3dd153a94a629cd64925371c9adad78
-  subpackages:
-  - hstore
-  - oid
-- name: github.com/magiconair/properties
-  version: 49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934
-- name: github.com/mattn/go-colorable
-  version: 5411d3eea5978e6cdc258b30de592b60df6aba96
-  repo: https://github.com/mattn/go-colorable
-- name: github.com/mattn/go-isatty
-  version: 57fdcb988a5c543893cc61bce354a6e24ab70022
-  repo: https://github.com/mattn/go-isatty
-- name: github.com/mitchellh/mapstructure
-  version: 06020f85339e21b2478f756a78e295255ffa4d6a
-- name: github.com/pelletier/go-toml
-  version: 0131db6d737cfbbfb678f8b7d92e55e27ce46224
-- name: github.com/spf13/afero
-  version: 57afd63c68602b63ed976de00dd066ccb3c319db
-  subpackages:
-  - mem
-- name: github.com/spf13/cast
-  version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4
-- name: github.com/spf13/cobra
-  version: 7b2c5ac9fc04fc5efafb60700713d4fa609b777b
-- name: github.com/spf13/jwalterweatherman
-  version: 12bd96e66386c1960ab0f74ced1362f66f552f7b
-- name: github.com/spf13/pflag
-  version: 4c012f6dcd9546820e378d0bdda4d8fc772cdfea
-- name: github.com/spf13/viper
-  version: 25b30aa063fc18e48662b86996252eabdcf2f0c7
-- name: golang.org/x/sys
-  version: e24f485414aeafb646f6fca458b0bf869c0880a1
-  repo: https://go.googlesource.com/sys
-  subpackages:
-  - unix
-- name: golang.org/x/text
-  version: e19ae1496984b1c655b8044a65c0300a3c878dd3
-  subpackages:
-  - transform
-  - unicode/norm
-- name: gopkg.in/yaml.v2
-  version: c95af922eae69f190717a0b7148960af8c55a072
-testImports: []

glide.yaml

@@ -1,14 +0,0 @@
-package: github.com/aquasecurity/kube-bench
-import:
-- package: github.com/fatih/color
-  version: ^1.5.0
-- package: github.com/golang/glog
-- package: github.com/jinzhu/gorm
-  version: ^1.0.0
-  subpackages:
-  - dialects/postgres
-- package: github.com/spf13/cobra
-  version: ^0.0.1
-- package: github.com/spf13/viper
-  version: ^1.0.0
-- package: gopkg.in/yaml.v2

images/kube-bench.svg

@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   viewBox="0 0 831.49597 755.90533"
+   height="755.90533"
+   width="831.49597"
+   xml:space="preserve"
+   id="svg2"
+   version="1.1"><metadata
+     id="metadata8"><rdf:RDF><cc:Work
+         rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" /></cc:Work></rdf:RDF></metadata><defs
+     id="defs6"><clipPath
+       id="clipPath22"
+       clipPathUnits="userSpaceOnUse"><path
+         id="path20"
+         d="M 0,566.929 H 623.622 V 0 H 0 Z" /></clipPath></defs><g
+     transform="matrix(1.3333333,0,0,-1.3333333,0,755.90533)"
+     id="g10"><g
+       transform="translate(314.8111,521.959)"
+       id="g12"><path
+         id="path14"
+         style="fill:#0ab1d5;fill-opacity:1;fill-rule:nonzero;stroke:none"
+         d="M 0,0 -106.784,-145.31 0,-280.384 105.477,-147.025 Z" /></g><g
+       id="g16"><g
+         clip-path="url(#clipPath22)"
+         id="g18"><g
+           transform="translate(51.8912,72.061)"
+           id="g24"><path
+             id="path26"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 v 71.061 c 0,3.629 2.86,6.6 6.6,6.6 3.74,0 6.6,-2.971 6.6,-6.6 V 32.45 h 2.97 c 1.32,0 2.42,0.551 3.52,1.981 L 33.44,52.69 c 1.43,1.981 3.081,3.3 5.72,3.3 3.63,0 6.271,-2.969 6.271,-6.599 0,-1.87 -0.881,-3.411 -1.981,-4.731 L 29.59,27.5 44.44,3.96 C 45.32,2.641 45.76,1.21 45.76,0 c 0,-3.63 -2.97,-6.6 -6.6,-6.6 -2.309,0 -4.4,1.54 -5.5,3.411 L 19.8,19.25 c -0.88,1.431 -1.98,2.091 -3.52,2.091 H 13.2 L 13.2,0 C 13.2,-3.63 10.34,-6.6 6.6,-6.6 2.86,-6.6 0,-3.63 0,0" /></g><g
+           transform="translate(104.9547,86.8013)"
+           id="g28"><path
+             id="path30"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 v 34.65 c 0,3.63 2.97,6.6 6.6,6.6 3.629,0 6.6,-2.97 6.6,-6.6 V 2.86 c 0,-8.47 3.409,-11.44 9.57,-11.44 4.73,0 9.24,2.86 11.33,4.95 v 38.28 c 0,3.63 2.97,6.6 6.6,6.6 3.63,0 6.6,-2.97 6.6,-6.6 v -50.16 c 0,-3.3 -2.53,-5.83 -5.72,-5.83 -2.97,0 -5.06,2.09 -5.72,4.95 l -0.55,2.42 C 32.12,-17.16 26.18,-21.34 18.149,-21.34 5.06,-21.34 0,-11.99 0,0" /></g><g
+           transform="translate(197.5084,90.4312)"
+           id="g32"><path
+             id="path34"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 v 12.65 c 0,8.47 -2.971,12.54 -10.341,12.54 -4.069,0 -8.029,-2.2 -10.559,-4.839 V -7.59 c 2.53,-2.639 6.49,-4.95 10.559,-4.95 C -2.971,-12.54 0,-8.47 0,0 m -34.101,-19.14 v 71.83 c 0,3.63 2.861,6.601 6.6,6.601 3.74,0 6.601,-2.971 6.601,-6.601 V 31.57 c 3.08,3.191 8.359,6.05 14.299,6.05 13.09,0 19.8,-8.8 19.8,-23.54 V -1.319 c 0,-14.741 -6.819,-23.651 -20.13,-23.651 -6.16,0 -11.88,2.97 -14.96,6.491 l -0.66,-2.201 c -0.769,-2.53 -3.08,-4.29 -5.72,-4.29 -3.299,0 -5.83,2.75 -5.83,5.83" /></g><g
+           transform="translate(251.7047,102.311)"
+           id="g36"><path
+             id="path38"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 c 0,9.57 -1.87,14.301 -9.9,14.301 -7.92,0 -9.9,-4.181 -9.9,-14.301 z M -33,-15.069 V 2.2 c 0,14.521 7.479,23.54 23.1,23.54 15.95,0 22.77,-8.689 22.77,-23.54 v -7.37 c 0,-2.859 -2.309,-5.17 -5.17,-5.17 h -27.5 v -5.939 c 0,-4.62 2.86,-9.13 10.89,-9.13 5.72,0 8.8,0.88 13.09,2.97 0.66,0.33 1.54,0.66 2.42,0.66 2.97,0 5.39,-2.42 5.39,-5.391 0,-2.309 -1.429,-3.96 -3.52,-5.17 -5.17,-2.97 -10.23,-4.51 -17.93,-4.51 -15.73,0 -23.54,8.25 -23.54,21.781" /></g><g
+           transform="translate(271.7564,99.4517)"
+           id="g40"><path
+             id="path42"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 c 0,3.3 2.53,5.83 5.721,5.83 h 19.91 c 3.3,0 5.83,-2.53 5.83,-5.83 0,-3.19 -2.53,-5.72 -5.83,-5.72 H 5.721 C 2.53,-5.72 0,-3.19 0,0" /></g><g
+           transform="translate(345.776,90.4312)"
+           id="g44"><path
+             id="path46"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 v 12.65 c 0,8.47 -2.971,12.54 -10.341,12.54 -4.069,0 -8.029,-2.2 -10.559,-4.839 V -7.59 c 2.53,-2.639 6.49,-4.95 10.559,-4.95 C -2.971,-12.54 0,-8.47 0,0 m -34.101,-19.14 v 71.83 c 0,3.63 2.861,6.601 6.6,6.601 3.74,0 6.601,-2.971 6.601,-6.601 V 31.57 c 3.08,3.191 8.359,6.05 14.299,6.05 13.09,0 19.8,-8.8 19.8,-23.54 V -1.319 c 0,-14.741 -6.819,-23.651 -20.13,-23.651 -6.16,0 -11.88,2.97 -14.96,6.491 l -0.66,-2.201 c -0.769,-2.53 -3.08,-4.29 -5.72,-4.29 -3.299,0 -5.83,2.75 -5.83,5.83" /></g><g
+           transform="translate(399.9723,102.311)"
+           id="g48"><path
+             id="path50"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 c 0,9.57 -1.87,14.301 -9.9,14.301 -7.92,0 -9.9,-4.181 -9.9,-14.301 z M -33,-15.069 V 2.2 c 0,14.521 7.479,23.54 23.1,23.54 15.95,0 22.77,-8.689 22.77,-23.54 v -7.37 c 0,-2.859 -2.309,-5.17 -5.17,-5.17 h -27.5 v -5.939 c 0,-4.62 2.86,-9.13 10.89,-9.13 5.72,0 8.8,0.88 13.09,2.97 0.66,0.33 1.54,0.66 2.42,0.66 2.97,0 5.39,-2.42 5.39,-5.391 0,-2.309 -1.429,-3.96 -3.52,-5.17 -5.17,-2.97 -10.23,-4.51 -17.93,-4.51 -15.73,0 -23.54,8.25 -23.54,21.781" /></g><g
+           transform="translate(421.8512,72.061)"
+           id="g52"><path
+             id="path54"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 v 50.16 c 0,3.301 2.53,5.83 5.72,5.83 2.97,0 5.06,-2.09 5.72,-4.949 l 0.55,-2.421 c 3.19,3.191 9.13,7.37 17.16,7.37 13.09,0 18.15,-9.349 18.15,-21.34 V 0 c 0,-3.63 -2.97,-6.6 -6.6,-6.6 -3.63,0 -6.599,2.97 -6.599,6.6 v 31.79 c 0,8.471 -3.411,11.44 -9.571,11.44 -4.73,0 -9.24,-2.86 -11.33,-4.95 L 13.2,0 C 13.2,-3.63 10.23,-6.6 6.6,-6.6 2.97,-6.6 0,-3.63 0,0" /></g><g
+           transform="translate(478.358,89.1118)"
+           id="g56"><path
+             id="path58"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 v 15.29 c 0,14.52 8.36,23.649 24.31,23.649 8.36,0 14.08,-3.08 18.15,-8.029 1.21,-1.54 1.87,-2.75 1.87,-4.511 0,-3.299 -2.53,-5.83 -5.83,-5.83 -1.76,0 -3.08,0.66 -4.4,1.981 -2.75,2.75 -5.39,4.62 -9.79,4.62 -8.69,0 -11.11,-5.83 -11.11,-12.981 L 13.2,1.1 c 0,-7.151 2.75,-12.981 11.44,-12.981 4.4,0 7.04,1.87 9.79,4.62 1.32,1.321 2.31,1.981 4.29,1.981 3.3,0 5.94,-2.531 5.94,-5.83 0,-1.76 -0.66,-2.97 -1.87,-4.51 C 38.72,-20.57 33,-23.65 24.64,-23.65 8.689,-23.65 0,-14.521 0,0" /></g><g
+           transform="translate(530.5396,72.061)"
+           id="g60"><path
+             id="path62"
+             style="fill:#464648;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 v 71.061 c 0,3.629 2.86,6.6 6.6,6.6 3.74,0 6.6,-2.971 6.6,-6.6 v -21.34 c 3.41,2.969 9.02,6.269 16.17,6.269 13.09,0 18.26,-9.349 18.26,-21.34 V 0 c 0,-3.63 -2.859,-6.6 -6.6,-6.6 -3.74,0 -6.6,2.97 -6.6,6.6 v 31.79 c 0,8.471 -3.52,11.44 -9.68,11.44 -4.729,0 -9.46,-2.86 -11.55,-4.95 V 0 C 13.2,-3.63 10.34,-6.6 6.6,-6.6 2.86,-6.6 0,-3.63 0,0" /></g><g
+           transform="translate(249.2096,192.0259)"
+           id="g64"><path
+             id="path66"
+             style="fill:#f1df36;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 h 0.008 l 131.211,0.031 h 0.013 c 3.063,0 6.107,0.66 8.916,1.863 L 65.602,49.549 -8.531,1.7 C -5.83,0.6 -2.923,0 0,0" /></g><g
+           transform="translate(420.2877,374.9341)"
+           id="g68"><path
+             id="path70"
+             style="fill:#faaf42;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 -105.477,-133.359 74.547,-47.655 c 3.392,1.452 6.439,3.697 8.747,6.559 l 75.104,93.431 6.686,8.317 c 1.38,1.714 2.479,3.637 3.289,5.675 0.384,0.965 0.701,1.954 0.95,2.962 z" /></g><g
+           transform="translate(145.3785,311.2251)"
+           id="g72"><path
+             id="path74"
+             style="fill:#faaf42;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 c 0.583,-2.568 1.609,-5.036 3.054,-7.245 0.401,-0.614 0.83,-1.209 1.285,-1.783 l 81.823,-101.735 c 2.396,-2.975 5.588,-5.289 9.138,-6.736 L 169.433,-69.65 62.648,65.424 Z" /></g><g
+           transform="translate(179.4977,457.7324)"
+           id="g76"><path
+             id="path78"
+             style="fill:#9ad7ec;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 c -2.408,-2.762 -4.144,-6.1 -4.985,-9.762 l -29.149,-126.8 c -0.65,-2.826 -0.715,-5.774 -0.239,-8.633 0.073,-0.44 0.155,-0.878 0.254,-1.312 l 62.648,65.424 z" /></g><g
+           transform="translate(484.1334,310.8643)"
+           id="g80"><path
+             id="path82"
+             style="fill:#9ad7ec;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="M 0,0 C 0.837,3.378 0.913,6.943 0.131,10.337 L -29.076,137.21 c -0.791,3.437 -2.374,6.586 -4.566,9.236 L -63.846,64.07 Z" /></g><g
+           transform="translate(317.7506,366.4487)"
+           id="g84"><path
+             id="path86"
+             style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="M 0,0 40.622,41.329 H 14.038 L -18.791,6.272 V 77.598 H -39.47 V -56.101 h 20.679 v 40.069 l 3.269,3.181 33.46,-43.25 h 27.03 z" /></g><g
+           transform="translate(275.7818,468.8486)"
+           id="g88"><path
+             id="path90"
+             style="fill:#1280c4;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 39.028,53.109 c -0.01,0 -0.022,10e-4 -0.033,10e-4 -0.047,0 -0.094,-0.003 -0.141,-0.003 C 38.521,53.105 38.187,53.099 37.853,53.082 37.814,53.08 37.776,53.072 37.738,53.07 34.783,52.909 31.86,52.166 29.192,50.889 L -89.022,-5.593 c -2.809,-1.342 -5.266,-3.235 -7.262,-5.523 L -67.755,-92.199 0,0.03 Z" /></g><g
+           transform="translate(442.8853,463.2578)"
+           id="g92"><path
+             id="path94"
+             style="fill:#1280c4;fill-opacity:1;fill-rule:nonzero;stroke:none"
+             d="m 0,0 -118.288,56.48 c -3.039,1.455 -6.412,2.215 -9.785,2.22 L -22.598,-88.324 7.606,-5.947 C 5.558,-3.467 2.978,-1.422 0,0" /></g></g></g></g></svg>