From b14d8fc9833e1eb1e7f9aa202eac0e82201c33fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Fri, 13 Dec 2019 16:34:55 +0100 Subject: [PATCH] [stable/datadog] Make the seccomp profile for system-probe optional (#19533) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [stable/datadog] Remove the seccomp profile for system-probe The `system-probe` container currently has a specific seccomp profile. This seccomp profile currently misses some syscalls that are necessary to exec inside the container. Concretely, attempting to exec inside the container produces this error: ``` $ kubectl exec -ti datadog-fswnc -c system-probe /bin/bash shell-init: error retrieving current directory: getcwd: cannot access parent directories: Operation not permitted bash: initialize_job_control: getpgrp failed: Operation not permitted command terminated with exit code 1 ``` If we add `setpgrp` to the seccomp profile, we get: ``` $ kubectl exec -ti datadog-kbg97 -c system-probe /bin/bash shell-init: error retrieving current directory: getcwd: cannot access parent directories: Operation not permitted I have no name!@datadog-kbg97:.$ exit ``` If we add `getcwd`, we get: ``` $ kubectl exec -ti datadog-7b7lf -c system-probe /bin/bash I have no name!@datadog-7b7lf:/$ exit ``` If we add `geteuid` and `geteuid32`, we get: ``` $ kubectl exec -ti datadog-c42rb -c system-probe /bin/bash /bin/bash: cannot set uid to -1: effective uid 0: Invalid argument /bin/bash: cannot set gid to -1: effective gid -1: Invalid argument bash-5.0$ exit ``` If we get `getgid` and `getgid32`, we get: ``` $ kubectl exec -ti datadog-tp4qd -c system-probe /bin/bash /bin/bash: cannot set uid to -1: effective uid 0: Invalid argument bash-5.0$ exit ``` etc. If we compare the seccomp profile of `system-probe` with the [default one](https://github.com/moby/moby/blob/4b0371fb36a958589319ab7c501ff4bc22645cfa/profiles/seccomp/default.json), we see that a lot of syscalls that are missing are innocuous (`getcwd`) or might become useful one day (`inotify` family) Some syscalls are added on purpose for the `system-probe` container like `bpf` or `perf_event_open` ones. But those syscalls are part of the [default seccomp profile for containers that have the `SYS_ADMIN` capability](https://github.com/moby/moby/blob/4b0371fb36a958589319ab7c501ff4bc22645cfa/profiles/seccomp/default.json#L567-L594), and the [`system-probe` container do have the `SYS_ADMIN` capability](https://github.com/helm/charts/blob/3907cebc7042f452506a7471f912d6d0c8380e51/stable/datadog/templates/container-system-probe.yaml#L7). So, the `system-probe` specific seccomp profile is not necessary to have the `system-probe` container able to load eBPF programs. Its removal has been tested on GKE, both with Ubuntu and with Container-Optimized OS and both with docker and containerd. Signed-off-by: Lénaïc Huard * Make the ad-hoc seccomp profile for system-probe an option which is enabled by default to stick with the current behavior. Signed-off-by: Lénaïc Huard * [stable/datadog] Allow use of any arbitrary seccomp profile …for system-probe. By default, it will create an ad-hoc one. Signed-off-by: Lénaïc Huard * [stable/datadog] Add a CI test for seccomp profile override Signed-off-by: Lénaïc Huard --- stable/datadog/Chart.yaml | 2 +- stable/datadog/README.md | 1 + stable/datadog/ci/no_hardened_seccomp.yaml | 5 +++++ stable/datadog/templates/daemonset.yaml | 14 +++++++++----- .../datadog/templates/system-probe-configmap.yaml | 2 ++ stable/datadog/values.yaml | 6 ++++++ 6 files changed, 24 insertions(+), 6 deletions(-) mode change 100755 => 100644 stable/datadog/Chart.yaml create mode 100644 stable/datadog/ci/no_hardened_seccomp.yaml diff --git a/stable/datadog/Chart.yaml b/stable/datadog/Chart.yaml old mode 100755 new mode 100644 index 66fda8d802..24f5749c30 --- a/stable/datadog/Chart.yaml +++ b/stable/datadog/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: datadog -version: 1.38.10 +version: 1.38.11 appVersion: "6" description: DataDog Agent keywords: diff --git a/stable/datadog/README.md b/stable/datadog/README.md index 0955fcbbb7..3db35ec030 100644 --- a/stable/datadog/README.md +++ b/stable/datadog/README.md @@ -372,6 +372,7 @@ helm install --name \ | `clusterchecksDeployment.rbac.serviceAccount` | existing ServiceAccount to use (ignored if rbac.create=true) for clusterchecks | `default` | | `clusterchecksDeployment.strategy` | Which update strategy to deploy the Cluster Checks Deployment | RollingUpdate with 0 maxUnavailable, 1 maxSurge | | `systemProbe.enabled` | If both this flag and `daemonset.useDedicatedContainers` are true, enable system probe collection | `false` | +| `systemProbe.seccomp` | Apply an ad-hoc seccomp profile to system-probe to restrict its privileges | `localhost/system-probe` | | `systemProbe.seccompRoot` | Seccomp root directory for system-probe | `/var/lib/kubelet/seccomp` | | `systemProbe.debugPort` | The port to expose pprof and expvar for system-probe agent, it is not enabled if the value is set to 0 | `0` | | `systemProbe.enableConntrack` | If true, system-probe connects to the netlink/conntrack subsystem to add NAT information to connection data. Ref: http://conntrack-tools.netfilter.org/| `true`| diff --git a/stable/datadog/ci/no_hardened_seccomp.yaml b/stable/datadog/ci/no_hardened_seccomp.yaml new file mode 100644 index 0000000000..4050954a15 --- /dev/null +++ b/stable/datadog/ci/no_hardened_seccomp.yaml @@ -0,0 +1,5 @@ +# Empty values file for testing default parameters. + +systemProbe: + enabled: true + seccomp: runtime/default diff --git a/stable/datadog/templates/daemonset.yaml b/stable/datadog/templates/daemonset.yaml index 9661677498..2281335850 100644 --- a/stable/datadog/templates/daemonset.yaml +++ b/stable/datadog/templates/daemonset.yaml @@ -44,7 +44,9 @@ spec: {{- end }} {{- if and .Values.systemProbe.enabled .Values.daemonset.useDedicatedContainers }} container.apparmor.security.beta.kubernetes.io/system-probe: {{ .Values.systemProbe.apparmor }} - container.seccomp.security.alpha.kubernetes.io/system-probe: localhost/system-probe + {{- if .Values.systemProbe.seccomp }} + container.seccomp.security.alpha.kubernetes.io/system-probe: {{ .Values.systemProbe.seccomp }} + {{- end }} {{- end }} {{- if .Values.daemonset.podAnnotations }} {{ toYaml .Values.daemonset.podAnnotations | indent 8 }} @@ -86,7 +88,7 @@ spec: {{- if .Values.daemonset.useDedicatedContainers }} initContainers: {{ include "containers-init" . | nindent 8 }} - {{- if .Values.systemProbe.enabled }} + {{- if and .Values.systemProbe.enabled (eq .Values.systemProbe.seccomp "localhost/system-probe") }} {{ include "system-probe-init" . | nindent 8 }} {{- end }} {{- end }} @@ -142,15 +144,17 @@ spec: name: passwd {{- end }} {{- if and .Values.systemProbe.enabled .Values.daemonset.useDedicatedContainers }} - - name: datadog-agent-security - configMap: - name: datadog-agent-security - name: sysprobe-config configMap: name: system-probe-config + {{- if eq .Values.systemProbe.seccomp "localhost/system-probe" }} + - name: datadog-agent-security + configMap: + name: datadog-agent-security - hostPath: path: {{ .Values.systemProbe.seccompRoot }} name: seccomp-root + {{- end }} - hostPath: path: /sys/kernel/debug name: debugfs diff --git a/stable/datadog/templates/system-probe-configmap.yaml b/stable/datadog/templates/system-probe-configmap.yaml index 731a34c464..e19dc129f4 100644 --- a/stable/datadog/templates/system-probe-configmap.yaml +++ b/stable/datadog/templates/system-probe-configmap.yaml @@ -23,6 +23,7 @@ data: sysprobe_socket: /opt/datadog-agent/run/sysprobe.sock enable_conntrack : {{ $.Values.systemProbe.enableConntrack }} bpf_debug: {{ $.Values.systemProbe.bpfDebug }} +{{- if eq .Values.systemProbe.seccomp "localhost/system-probe" }} --- apiVersion: v1 kind: ConfigMap @@ -197,3 +198,4 @@ data: ] } {{- end }} +{{- end }} diff --git a/stable/datadog/values.yaml b/stable/datadog/values.yaml index 86d26cea56..40baa5450e 100644 --- a/stable/datadog/values.yaml +++ b/stable/datadog/values.yaml @@ -307,6 +307,12 @@ systemProbe: # enableConntrack: true + ## @param seccomp - string - required + ## Apply an ad-hoc seccomp profile to the system-probe agent to restrict its privileges + ## Note that this will break `kubectl exec … -c system-probe -- /bin/bash` + # + seccomp: localhost/system-probe + ## @param seccompRoot - string - required ## Specify the seccomp profile root directory #