Adding a section for Azure Kubernetes Service (#495)

* Adding a section for Azure Kubernetes Service

steps to run kube bench on AKS worker nodes

* Update README.md

* Update README.md

Co-authored-by: Roberto Rojas <robertojrojas@gmail.com>
Co-authored-by: Liz Rice <liz@lizrice.com>

.goreleaser.yml

@@ -1,5 +1,6 @@
 env:
   - GO111MODULE=on
+  - KUBEBENCH_CFG=/etc/kube-bench/cfg
 builds:
   - main: main.go
     binary: kube-bench
@@ -7,14 +8,21 @@ builds:
       - linux
     goarch:
       - amd64
+    ldflags:
+     - "-X github.com/aquasecurity/kube-bench/cmd.KubeBenchVersion={{.Version}}"
+     - "-X github.com/aquasecurity/kube-bench/cmd.cfgDir={{.Env.KUBEBENCH_CFG}}"
 # Archive customization
 archive:
   format: tar.gz
+  files:
+    - "cfg/**/*"
 nfpm:
   vendor: Aqua Security
   description: "The Kubernetes Bench for Security is a Go application that checks whether Kubernetes is deployed according to security best practices"
   license: Apache-2.0
   homepage: https://github.com/aquasecurity/kube-bench
+  files:
+    "cfg/**/*": "/etc/kube-bench/cfg"
   formats:
     - deb
     - rpm

Dockerfile

@@ -12,6 +12,11 @@ WORKDIR /opt/kube-bench/
 # add GNU ps for -C, -o cmd, and --no-headers support
 # https://github.com/aquasecurity/kube-bench/issues/109
 RUN apk --no-cache add procps
+
+# Openssl is used by OpenShift tests
+# https://github.com/aquasecurity/kube-bench/issues/535
+RUN apk --no-cache add openssl
+
 COPY --from=build /go/bin/kube-bench /usr/local/bin/kube-bench
 COPY entrypoint.sh .
 COPY cfg/ cfg/

README.md

@@ -19,26 +19,27 @@ Tests are configured with YAML files, making this tool easy to update as test sp
 Table of Contents
 =================
 
-- [Table of Contents](#table-of-contents)
-  - [CIS Kubernetes Benchmark support](#cis-kubernetes-benchmark-support)
-  - [Installation](#installation)
-  - [Running kube-bench](#running-kube-bench)
-    - [Running inside a container](#running-inside-a-container)
-    - [Running in a Kubernetes cluster](#running-in-a-kubernetes-cluster)
-    - [Running in an EKS cluster](#running-in-an-eks-cluster)
-    - [Installing from a container](#installing-from-a-container)
-    - [Installing from sources](#installing-from-sources)
-  - [Running on OpenShift](#running-on-openshift)
-  - [Output](#output)
-  - [Configuration](#configuration)
-  - [Test config YAML representation](#test-config-yaml-representation)
-    - [Omitting checks](#omitting-checks)
-  - [Roadmap](#roadmap)
-  - [Testing locally with kind](#testing-locally-with-kind)
-  - [Contributing](#contributing)
-    - [Bugs](#bugs)
-    - [Features](#features)
-    - [Pull Requests](#pull-requests)
+* [CIS Kubernetes Benchmark support](#cis-kubernetes-benchmark-support)
+* [Installation](#installation)
+* [Running kube-bench](#running-kube-bench)
+  * [Running inside a container](#running-inside-a-container)
+  * [Running in a kubernetes cluster](#running-in-a-kubernetes-cluster)
+  * [Running in an Azure Kubernetes Service(AKS) cluster](#running-in-an-aks-cluster)
+  * [Running in an EKS cluster](#running-in-an-eks-cluster)
+  * [Installing from a container](#installing-from-a-container)
+  * [Installing from sources](#installing-from-sources)
+* [Running on OpenShift](#running-on-openshift)
+* [Output](#output)
+* [Configuration](#configuration)
+* [Test config YAML representation](#test-config-yaml-representation)
+  * [Omitting checks](#omitting-checks)
+* [Roadmap](#roadmap)
+* [Testing locally with kind](#testing-locally-with-kind)
+* [Contributing](#contributing)
+  * [Bugs](#bugs)
+  * [Features](#features)
+  * [Pull Requests](#pull-requests)
+
       
 ## CIS Kubernetes Benchmark support
 
@@ -177,6 +178,25 @@ To run the tests on the master node, the pod needs to be scheduled on that node.
 
 The default labels applied to master nodes has changed since Kubernetes 1.11, so if you are using an older version you may need to modify the nodeSelector and tolerations to run the job on the master node.
 
+
+### Running in an AKS cluster
+
+1. Create an AKS cluster(e.g. 1.13.7) with RBAC enabled, otherwise there would be 4 failures
+
+1. Use the [kubectl-enter plugin] (https://github.com/kvaps/kubectl-enter) to shell into a node 
+`
+kubectl-enter {node-name}
+` 
+or ssh to one agent node
+could open nsg 22 port and assign a public ip for one agent node (only for testing purpose)
+
+1. Run CIS benchmark to view results:
+```
+docker run --rm -v `pwd`:/host aquasec/kube-bench:latest install
+./kube-bench node
+```
+kube-bench cannot be run on AKS master nodes 
+
 ### Running in an EKS cluster
 
 There is a `job-eks.yaml` file for running the kube-bench node checks on an EKS cluster. The significant difference on EKS is that it's not possible to schedule jobs onto the master node, so master checks can't be performed
@@ -190,10 +210,10 @@ aws ecr create-repository --repository-name k8s/kube-bench --image-tag-mutabilit
 3. Download, build and push the kube-bench container image to your ECR repo
 ```
 git clone https://github.com/aquasecurity/kube-bench.git
+cd kube-bench
 $(aws ecr get-login --no-include-email --region <AWS_REGION>)
 docker build -t k8s/kube-bench .
 docker tag k8s/kube-bench:latest <AWS_ACCT_NUMBER>.dkr.ecr.<AWS_REGION>.amazonaws.com/k8s/kube-bench:latest
-docker tag k8s/kube-bench:latest <AWS_ACCT_NUMBER>.dkr.ecr.<AWS_REGION>.amazonaws.com/k8s/kube-bench:latest
 docker push <AWS_ACCT_NUMBER>.dkr.ecr.<AWS_REGION>.amazonaws.com/k8s/kube-bench:latest
 ```
 4. Copy the URI of your pushed image, the URI format is like this: `<AWS_ACCT_NUMBER>.dkr.ecr.<AWS_REGION>.amazonaws.com/k8s/kube-bench:latest`

cfg/cis-1.3/node.yaml

@@ -423,7 +423,7 @@ groups:
     remediation: |
       Run the below command (based on the file location on your system) on the each worker
       node. For example,
-      chmod 755 $kubeletsvc
+      chmod 644 $kubeletsvc
     scored: true
 
   - id: 2.2.4

cfg/cis-1.5/node.yaml

@@ -32,7 +32,7 @@ groups:
     remediation: |
       Run the below command (based on the file location on your system) on the each worker node. 
       For example, 
-      chmod 755 $kubeletsvc 
+      chmod 644 $kubeletsvc 
     scored: true
 
   - id: 4.1.2

cfg/config.yaml

@@ -172,4 +172,4 @@ version_mapping:
   "1.16": "cis-1.5"
   "1.17": "cis-1.5"
   "ocp-3.10": "rh-0.7"
-  "ocp-3.11": "rh-0.7"
+  "ocp-3.11": "rh-0.7"

cfg/node_only.yaml

@@ -59,4 +59,15 @@ node:
     svc:
       - "/lib/systemd/system/kube-proxy.service"
     defaultconf: /etc/kubernetes/addons/kube-proxy-daemonset.yaml
-    defaultkubeconfig: "/etc/kubernetes/proxy.conf"
+    defaultkubeconfig: "/etc/kubernetes/proxy.conf"
+
+version_mapping:
+  "1.11": "cis-1.3"
+  "1.12": "cis-1.3"
+  "1.13": "cis-1.4"
+  "1.14": "cis-1.4"
+  "1.15": "cis-1.5"
+  "1.16": "cis-1.5"
+  "1.17": "cis-1.5"
+  "ocp-3.10": "rh-0.7"
+  "ocp-3.11": "rh-0.7"

cfg/rh-0.7/config.yaml

@@ -10,18 +10,23 @@ master:
   scheduler:
     bins:
       - "openshift start master controllers"
+      - "hyperkube kube-scheduler"
     confs:
       - /etc/origin/master/scheduler.json
 
   controllermanager:
     bins:
       - "openshift start master controllers"
+      - "hypershift openshift-controller-manager"
 
   etcd:
     bins:
       - openshift start etcd
 
 node:
+  svcs:
+    - /etc/systemd/system/atomic-openshift-node.service
+    - /etc/systemd/system/origin-node.service
   proxy:
     bins:
       - openshift start network

cfg/rh-0.7/node.yaml

@@ -254,7 +254,7 @@ groups:
 
   - id: 8.3
     text: "Verify the kubelet service file permissions of 644"
-    audit: "stat -c %a /etc/systemd/system/atomic-openshift-node.service"
+    audit: "stat -c %a $nodesvc"
     tests:
       bin_op: or
       test_items:
@@ -275,12 +275,12 @@ groups:
           set: true
     remediation: |
       Run the below command on each worker node.
-      chmod 644 /etc/systemd/system/atomic-openshift-node.service
+      chmod 644 $nodesvc
     scored: true
 
   - id: 8.4
     text: "Verify the kubelet service file ownership of root:root"
-    audit: "stat -c %U:%G /etc/systemd/system/atomic-openshift-node.service"
+    audit: "stat -c %U:%G $nodesvc"
     tests:
       test_items:
         - flag: "root:root"
@@ -290,7 +290,7 @@ groups:
           set: true
       remediation: |
         Run the below command on each worker node.
-        chown root:root /etc/systemd/system/atomic-openshift-node.service
+        chown root:root $nodesvc
       scored: true
 
   - id: 8.5

check/check.go

@@ -171,7 +171,6 @@ func (c *Check) run() State {
 		c.State = PASS
 		c.ActualValue = finalOutput.actualResult
 		c.ExpectedResult = finalOutput.ExpectedResult
-		glog.V(3).Infof("Check.ID: %s Command: %q TestResult: %t Score: %q \n", c.ID, lastCommand, finalOutput.testResult, c.State)
 	} else {
 		if c.Scored {
 			c.State = FAIL
@@ -180,7 +179,9 @@ func (c *Check) run() State {
 		}
 	}
 
-	if finalOutput == nil {
+	if finalOutput != nil {
+		glog.V(3).Infof("Check.ID: %s Command: %q TestResult: %t State: %q \n", c.ID, lastCommand, finalOutput.testResult, c.State)
+	} else {
 		glog.V(3).Infof("Check.ID: %s Command: %q TestResult: <<EMPTY>> \n", c.ID, lastCommand)
 	}
 
@@ -242,8 +243,7 @@ func isShellCommand(s string) bool {
 
 	out, err := cmd.Output()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s\n", err)
-		os.Exit(1)
+		exitWithError(fmt.Errorf("failed to check if command: %q is valid %v", s, err))
 	}
 
 	if strings.Contains(string(out), s) {
@@ -331,6 +331,13 @@ func runExecCommands(audit string, commands []*exec.Cmd, out *bytes.Buffer) (Sta
 		i++
 	}
 
-	glog.V(3).Infof("Command %q - Output:\n\n %s\n", audit, out.String())
+	glog.V(3).Infof("Command %q - Output:\n\n %q\n - Error Messages:%q \n", audit, out.String(), errmsgs)
 	return "", errmsgs
 }
+
+func exitWithError(err error) {
+	fmt.Fprintf(os.Stderr, "\n%v\n", err)
+	// flush before exit non-zero
+	glog.Flush()
+	os.Exit(1)
+}

check/test.go

@@ -165,7 +165,7 @@ func compareOp(tCompareOp string, flagVal string, tCompareValue string) (string,
 	case "gt", "gte", "lt", "lte":
 		a, b, err := toNumeric(flagVal, tCompareValue)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "%v\n", err)
+			fmt.Fprintf(os.Stderr, "Not numeric value - flag: %q - compareValue: %q %v\n", flagVal, tCompareValue, err)
 			os.Exit(1)
 		}
 		switch tCompareOp {

cmd/common.go

@@ -90,7 +90,7 @@ func runChecks(nodetype check.NodeType, testYamlFile string) {
 
 	// Checks that the executables we need for the section are running.
 	if err != nil {
-		exitWithError(err)
+		exitWithError(fmt.Errorf("failed to get a set of executables needed for tests: %v", err))
 	}
 
 	confmap := getFiles(typeConf, "config")
@@ -229,7 +229,7 @@ func loadConfig(nodetype check.NodeType) string {
 
 	benchmarkVersion, err := getBenchmarkVersion(kubeVersion, benchmarkVersion, viper.GetViper())
 	if err != nil {
-		exitWithError(err)
+		exitWithError(fmt.Errorf("failed to get benchMark version: %v", err))
 	}
 
 	path, err := getConfigFilePath(benchmarkVersion, file)
@@ -319,6 +319,7 @@ func getBenchmarkVersion(kubeVersion, benchmarkVersion string, v *viper.Viper) (
 
 // isMaster verify if master components are running on the node.
 func isMaster() bool {
+	loadConfig(check.MASTER)
 	return isThisNodeRunning(check.MASTER)
 }
 

cmd/common_test.go

@@ -155,6 +155,20 @@ func TestIsMaster(t *testing.T) {
 			isMaster: false,
 		},
 	}
+	cfgDirOld := cfgDir
+	cfgDir = "../cfg"
+	defer func() {
+		cfgDir = cfgDirOld
+	}()
+
+	execCode := `#!/bin/sh
+	echo "Server Version: v1.13.10"
+	`
+	restore, err := fakeExecutableInPath("kubectl", execCode)
+	if err != nil {
+		t.Fatal("Failed when calling fakeExecutableInPath ", err)
+	}
+	defer restore()
 
 	for _, tc := range testCases {
 		cfgFile = tc.cfgFile
@@ -386,6 +400,73 @@ func TestValidTargets(t *testing.T) {
 	}
 }
 
+func TestIsEtcd(t *testing.T) {
+	testCases := []struct {
+		name            string
+		cfgFile         string
+		getBinariesFunc func(*viper.Viper, check.NodeType) (map[string]string, error)
+		isEtcd          bool
+	}{
+		{
+			name:    "valid config, is etcd and all components are running",
+			cfgFile: "../cfg/config.yaml",
+			getBinariesFunc: func(viper *viper.Viper, nt check.NodeType) (strings map[string]string, i error) {
+				return map[string]string{"etcd": "etcd"}, nil
+			},
+			isEtcd: true,
+		},
+		{
+			name:    "valid config, is etcd and but not all components are running",
+			cfgFile: "../cfg/config.yaml",
+			getBinariesFunc: func(viper *viper.Viper, nt check.NodeType) (strings map[string]string, i error) {
+				return map[string]string{}, nil
+			},
+			isEtcd: false,
+		},
+		{
+			name:    "valid config, is etcd, not all components are running and fails to find all binaries",
+			cfgFile: "../cfg/config.yaml",
+			getBinariesFunc: func(viper *viper.Viper, nt check.NodeType) (strings map[string]string, i error) {
+				return map[string]string{}, errors.New("failed to find binaries")
+			},
+			isEtcd: false,
+		},
+		{
+			name:    "valid config, does not include etcd",
+			cfgFile: "../cfg/node_only.yaml",
+			isEtcd:  false,
+		},
+	}
+	cfgDirOld := cfgDir
+	cfgDir = "../cfg"
+	defer func() {
+		cfgDir = cfgDirOld
+	}()
+
+	execCode := `#!/bin/sh
+	echo "Server Version: v1.15.03"
+	`
+	restore, err := fakeExecutableInPath("kubectl", execCode)
+	if err != nil {
+		t.Fatal("Failed when calling fakeExecutableInPath ", err)
+	}
+	defer restore()
+
+	for _, tc := range testCases {
+		cfgFile = tc.cfgFile
+		initConfig()
+
+		oldGetBinariesFunc := getBinariesFunc
+		getBinariesFunc = tc.getBinariesFunc
+		defer func() {
+			getBinariesFunc = oldGetBinariesFunc
+			cfgFile = ""
+		}()
+
+		assert.Equal(t, tc.isEtcd, isEtcd(), tc.name)
+	}
+}
+
 func loadConfigForTest() (*viper.Viper, error) {
 	viperWithData := viper.New()
 	viperWithData.SetConfigFile(filepath.Join("..", cfgDir, "config.yaml"))
@@ -410,11 +491,6 @@ func fakeExecutableInPath(execFile, execCode string) (restoreFn, error) {
 		return nil, err
 	}
 
-	err = os.Chdir(tmp)
-	if err != nil {
-		return nil, err
-	}
-
 	if len(execCode) > 0 {
 		ioutil.WriteFile(filepath.Join(tmp, execFile), []byte(execCode), 0700)
 	} else {

cmd/root.go

@@ -38,7 +38,7 @@ var (
 	kubeVersion        string
 	benchmarkVersion   string
 	cfgFile            string
-	cfgDir             string
+	cfgDir             = "./cfg/"
 	jsonFmt            bool
 	junitFmt           bool
 	pgSQL              bool
@@ -64,7 +64,7 @@ var RootCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		benchmarkVersion, err := getBenchmarkVersion(kubeVersion, benchmarkVersion, viper.GetViper())
 		if err != nil {
-			exitWithError(err)
+			exitWithError(fmt.Errorf("unable to determine benchmark version: %v", err))
 		}
 
 		if isMaster() {
@@ -81,7 +81,7 @@ var RootCmd = &cobra.Command{
 
 		// Etcd is only valid for CIS 1.5 and later,
 		// this a gatekeeper for previous versions.
-		if isEtcd() && validTargets(benchmarkVersion, []string{string(check.ETCD)}) {
+		if validTargets(benchmarkVersion, []string{string(check.ETCD)}) && isEtcd() {
 			glog.V(1).Info("== Running etcd checks ==\n")
 			runChecks(check.ETCD, loadConfig(check.ETCD))
 		}
@@ -145,7 +145,7 @@ func init() {
 		`Run all the checks under this comma-delimited list of groups. Example --group="1.1"`,
 	)
 	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is ./cfg/config.yaml)")
-	RootCmd.PersistentFlags().StringVarP(&cfgDir, "config-dir", "D", "./cfg/", "config directory")
+	RootCmd.PersistentFlags().StringVarP(&cfgDir, "config-dir", "D", cfgDir, "config directory")
 	RootCmd.PersistentFlags().StringVar(&kubeVersion, "version", "", "Manually specify Kubernetes version, automatically detected if unset")
 	RootCmd.PersistentFlags().StringVar(&benchmarkVersion, "benchmark", "", "Manually specify CIS benchmark version. It would be an error to specify both --version and --benchmark flags")
 

cmd/run.go

@@ -29,12 +29,12 @@ var runCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		targets, err := cmd.Flags().GetStringSlice("targets")
 		if err != nil {
-			exitWithError(err)
+			exitWithError(fmt.Errorf("unable to get `targets` from command line :%v", err))
 		}
 
 		benchmarkVersion, err := getBenchmarkVersion(kubeVersion, benchmarkVersion, viper.GetViper())
 		if err != nil {
-			exitWithError(err)
+			exitWithError(fmt.Errorf("unable to get benchmark version. error: %v", err))
 		}
 
 		glog.V(2).Infof("Checking targets %v for %v", targets, benchmarkVersion)