From f599462ad7caa05ee921a7cc8691e1a49cd3257b Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Mon, 6 Aug 2018 11:51:24 -0500 Subject: [PATCH 1/6] Add a short chapter about network policies I will then expand this chapter to add examples showing how to isolate namespaces; but let's start with that. --- k8s/netpol-allow-testcurl-for-testweb.yaml | 14 ++ k8s/netpol-deny-all-for-testweb.yaml | 10 + slides/kube-fullday.yml | 9 +- slides/kube-halfday.yml | 2 + slides/kube-selfpaced.yml | 1 + slides/kube/netpol.md | 235 +++++++++++++++++++++ 6 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 k8s/netpol-allow-testcurl-for-testweb.yaml create mode 100644 k8s/netpol-deny-all-for-testweb.yaml create mode 100644 slides/kube/netpol.md diff --git a/k8s/netpol-allow-testcurl-for-testweb.yaml b/k8s/netpol-allow-testcurl-for-testweb.yaml new file mode 100644 index 00000000..c0a73f13 --- /dev/null +++ b/k8s/netpol-allow-testcurl-for-testweb.yaml @@ -0,0 +1,14 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-testcurl-for-testweb +spec: + podSelector: + matchLabels: + run: testweb + ingress: + - from: + - podSelector: + matchLabels: + run: testcurl + diff --git a/k8s/netpol-deny-all-for-testweb.yaml b/k8s/netpol-deny-all-for-testweb.yaml new file mode 100644 index 00000000..e975991c --- /dev/null +++ b/k8s/netpol-deny-all-for-testweb.yaml @@ -0,0 +1,10 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: deny-all-for-testweb +spec: + podSelector: + matchLabels: + run: testweb + ingress: [] + diff --git a/slides/kube-fullday.yml b/slides/kube-fullday.yml index 98b0803b..173668d8 100644 --- a/slides/kube-fullday.yml +++ b/slides/kube-fullday.yml @@ -38,10 +38,11 @@ chapters: - - k8s/kubectlscale.md - k8s/daemonset.md - k8s/rollout.md - #- k8s/logs-cli.md - #- k8s/logs-centralized.md - #- k8s/helm.md - #- k8s/namespaces.md + - k8s/logs-cli.md + - k8s/logs-centralized.md + - k8s/helm.md + - k8s/namespaces.md + - k8s/netpol.md - k8s/whatsnext.md - k8s/links.md - shared/thankyou.md diff --git a/slides/kube-halfday.yml b/slides/kube-halfday.yml index 01b9dd86..9df37a3c 100644 --- a/slides/kube-halfday.yml +++ b/slides/kube-halfday.yml @@ -42,9 +42,11 @@ chapters: - k8s/rollout.md - - k8s/logs-cli.md # Bridget hasn't added EFK yet +<<<<<<< HEAD #- k8s/logs-centralized.md - k8s/helm.md - k8s/namespaces.md + #- k8s/netpol.md - k8s/whatsnext.md # - k8s/links.md # Bridget-specific diff --git a/slides/kube-selfpaced.yml b/slides/kube-selfpaced.yml index 190af30a..00f7f97f 100644 --- a/slides/kube-selfpaced.yml +++ b/slides/kube-selfpaced.yml @@ -41,6 +41,7 @@ chapters: - k8s/logs-centralized.md - k8s/helm.md - k8s/namespaces.md + - k8s/netpol.md - k8s/whatsnext.md - k8s/links.md - shared/thankyou.md diff --git a/slides/kube/netpol.md b/slides/kube/netpol.md new file mode 100644 index 00000000..a29739a6 --- /dev/null +++ b/slides/kube/netpol.md @@ -0,0 +1,235 @@ +# Network policies + +- Namespaces help us to *organize* resources + +- Namespaces do not provide isolation + +- By default, every pod can contact every other pod + +- By default, every service accepts traffic from anyone + +- If we want this to be different, we need *network policies* + +--- + +## What's a network policy? + +A network policy is defined by the following things. + +- A *pod selector* indicating which pods it applies to + + e.g.: "all pods in namespace `blue` with the label `zone=internal`" + +- A list of *ingress rules* indicating which inbound traffic is allowed + + e.g.: "TCP connections to ports 8000 and 8080 coming from pods with label `zone=dmz`, + and from the external subnet 4.42.6.0/24, except 4.42.6.5" + +- A list of *egress rules* indicating which outbound traffic is allowed + +A network policy can provide ingress rules, egress rules, or both. + +--- + +## How do network policies apply? + +- A pod can be "selected" by any number of network policies + +- If a pod isn't selected by any network policy, then its traffic is unrestricted + + (Except if it's traffic to/from another pod, which itself is selected and restricted) + +- If a pod is selected by at least one network policy, then all traffic is blocked ... + + ... unless it is explicitly allowed by one of these network policies + +Note: that logic applies separately to ingress and egress traffic. + +If a pod is selected by network policies that only specify ingress rules, +
then egress traffic for that pod is unrestricted. + +--- + +## The rationale for network policies + +- In network security, it is generally considered better to "deny all, then allow selectively" + + (The other approach, "allow all, then block selectively" makes it too easy to leave holes) + +- As soon as one network policy selects a pod, the pod enters this "deny all" logic + +- Further network policies can open additional access + +- Good network policies should be scoped as precisely as possible + +- In particular: make sure that the selector is not too broad + + (Otherwise, you end up affecting pods that were otherwise well secured) + +--- + +## Our first network policy + +This is our game plan: + +- run a web server in a pod + +- create a network policy to block all access to the web server + +- create another network policy to allow access only from specific pods + +--- + +## Running our test web server + +.exercise[ + +- Let's use the `nginx` image: + ```bash + kubectl run testweb --image=nginx + ``` + +- Find out the IP address of the pod with one of these two commands: + ```bash + kubectl get pods -o wide -l run=testweb + IP=$(kubectl get pods -l run=testweb -o json | jq -r .items[0].status.podIP) + ``` + +- Check that we can connect to the server: + ```bash + curl $IP + ``` +] + +The `curl` command should show us the "Welcome to nginx!" page. + +--- + +## Adding a very restrictive network policy + +- The policy will select pods with the label `run=testweb` + +- It will specify an empty list of ingress rules (matching nothing) + +.exercise[ + +- Apply the policy in this YAML file: + ```bash + kubectl apply -f ~/container.training/k8s/netpol-deny-all-for-testweb.yaml + ``` + +- Check if we can still access the server: + ```bash + curl $IP + ``` + +] + +The `curl` command should now time out. + +--- + +## Looking at the network policy + +This is the file that we applied: + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: deny-all-for-testweb +spec: + podSelector: + matchLabels: + run: testweb + ingress: [] +``` + +--- + +## Allowing connections only from specific pods + +- We want to allow traffic from pods with the label `run=testcurl` + +- Reminder: this label is automatically applied when we do `kubectl run testcurl ...` + +.exercise[ + +- Apply another policy: + ```bash + kubectl apply -f ~/container.training/netpol-allow-testcurl-for-testweb.yaml + ``` + +] + +--- + +## Looking at the network policy + +This is the second file that we applied: + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-testcurl-for-testweb +spec: + podSelector: + matchLabels: + run: testweb + ingress: + - from: + - podSelector: + matchLabels: + run: testcurl +``` + +--- + +## Testing the network policy + +- Let's create pods with, and without, the required label + +.exercise[ + +- Try to connect to testweb from a pod with the `run=testcurl` label: + ```bash + kubectl run testcurl --rm -i --image=centos -- curl -m3 $IP + ``` + +- Try to connect to testweb with a different label: + ```bash + kubectl run testkurl --rm -i --image=centos -- curl -m3 $IP + ``` + +] + +The first command will work (and show the "Welcome to nginx!" page). + +The second command will fail and time out after 3 seconds. + +(The timeout is obtained with the `-m3` option.) + +--- + +## An important warning + +- Some network plugins only have partial support for network policies + +- For instance, Weave [doesn't support ipBlock (yet)](https://github.com/weaveworks/weave/issues/3168) + +- Weave added support for egress rules [in version 2.4](https://github.com/weaveworks/weave/pull/3313) (released in July 2018) + +- Unsupported features might be silently ignored + + (Making you believe that you are secure, when you're not) + +--- + +## Further resources + +- As always, the [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/) is a good starting point + +- [Ahmet Alp Balkan](https://ahmet.im/) delivered a [very good talk about network policies](https://www.youtube.com/watch?list=PLj6h78yzYM2P-3-xqvmWaZbbI1sW-ulZb&v=3gGpMmYeEO8) at KubeCon + +- He also compiled some [ready-to-use recipes](https://github.com/ahmetb/kubernetes-network-policy-recipes) for network policies From c00c87f8f21e5b3792a4a3622a9769043aa29b5e Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sun, 19 Aug 2018 09:37:18 -0500 Subject: [PATCH 2/6] Clarify network policies Add clarification re/ pod-to-pod traffic. Explain that it's stateful (which most people would expect anyway). --- slides/kube/netpol.md | 44 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/slides/kube/netpol.md b/slides/kube/netpol.md index a29739a6..ab813955 100644 --- a/slides/kube/netpol.md +++ b/slides/kube/netpol.md @@ -37,16 +37,46 @@ A network policy can provide ingress rules, egress rules, or both. - If a pod isn't selected by any network policy, then its traffic is unrestricted - (Except if it's traffic to/from another pod, which itself is selected and restricted) + (In other words: in the absence of network policies, all traffic is allowed) - If a pod is selected by at least one network policy, then all traffic is blocked ... ... unless it is explicitly allowed by one of these network policies -Note: that logic applies separately to ingress and egress traffic. +--- -If a pod is selected by network policies that only specify ingress rules, -
then egress traffic for that pod is unrestricted. +class: extra-details + +## Traffic filtering is flow-oriented + +- Network policies deal with *connections*, not individual packets + +- Example: to allow HTTP (80/tcp) connections to pod A, you only need an ingress rule + + (You do not need a matching egress rule to allow response traffic to go through) + +- This also stands for UDP traffic + + (Allowing DNS traffic can be done with a single rule) + +- Network policy implementations use stateful connection tracking + +--- + +## Pod-to-pod traffic + +- Connections from pod A to pod B have to be allowed by both pods: + + - pod A has to be unrestricted, or allow the connection as an *egress* rule + + - pod B has to be unrestricted, or allow the connection as an *ingress* rule + +- As a consequence: if a network policy restricts traffic going from/to a pod, +
+ the restriction cannot be overridden by a network policy selecting another pod + +- This prevents an entity with access to namespace A (but no access to namespace B) + from adding network policies giving them access to namespace B --- @@ -230,6 +260,8 @@ The second command will fail and time out after 3 seconds. - As always, the [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/) is a good starting point -- [Ahmet Alp Balkan](https://ahmet.im/) delivered a [very good talk about network policies](https://www.youtube.com/watch?list=PLj6h78yzYM2P-3-xqvmWaZbbI1sW-ulZb&v=3gGpMmYeEO8) at KubeCon +- And two resources by [Ahmet Alp Balkan](https://ahmet.im/): -- He also compiled some [ready-to-use recipes](https://github.com/ahmetb/kubernetes-network-policy-recipes) for network policies + - a [very good talk about network policies](https://www.youtube.com/watch?list=PLj6h78yzYM2P-3-xqvmWaZbbI1sW-ulZb&v=3gGpMmYeEO8) at KubeCon North America 2017 + + - a repository of [ready-to-use recipes](https://github.com/ahmetb/kubernetes-network-policy-recipes) for network policies From 7d7cb0eadb90567fd36f4011b5054e1634afde3e Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Tue, 21 Aug 2018 04:21:39 -0500 Subject: [PATCH 3/6] Put netpol file in the right directory --- slides/{kube => k8s}/netpol.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename slides/{kube => k8s}/netpol.md (100%) diff --git a/slides/kube/netpol.md b/slides/k8s/netpol.md similarity index 100% rename from slides/kube/netpol.md rename to slides/k8s/netpol.md From 83b213357397b7f643517d899d934c8ce1668fcc Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Thu, 23 Aug 2018 04:56:22 -0500 Subject: [PATCH 4/6] Oops, fixing bad conflict resolve --- slides/kube-halfday.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/slides/kube-halfday.yml b/slides/kube-halfday.yml index 9df37a3c..be3a12a5 100644 --- a/slides/kube-halfday.yml +++ b/slides/kube-halfday.yml @@ -42,7 +42,6 @@ chapters: - k8s/rollout.md - - k8s/logs-cli.md # Bridget hasn't added EFK yet -<<<<<<< HEAD #- k8s/logs-centralized.md - k8s/helm.md - k8s/namespaces.md From dc0850ef3eaff76f75976c261cd944f3b91d7df5 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Mon, 27 Aug 2018 11:36:46 -0500 Subject: [PATCH 5/6] Expand the network policy section --- k8s/netpol-dockercoins.yaml | 22 +++++ slides/k8s/netpol.md | 155 +++++++++++++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 k8s/netpol-dockercoins.yaml diff --git a/k8s/netpol-dockercoins.yaml b/k8s/netpol-dockercoins.yaml new file mode 100644 index 00000000..79a78f97 --- /dev/null +++ b/k8s/netpol-dockercoins.yaml @@ -0,0 +1,22 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: deny-from-other-namespaces +spec: + podSelector: + matchLabels: + ingress: + - from: + - podSelector: {} +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-webui +spec: + podSelector: + matchLabels: + run: webui + ingress: + - from: [] + diff --git a/slides/k8s/netpol.md b/slides/k8s/netpol.md index ab813955..7049570b 100644 --- a/slides/k8s/netpol.md +++ b/slides/k8s/netpol.md @@ -98,7 +98,7 @@ class: extra-details --- -## Our first network policy +## Our first network policies This is our game plan: @@ -256,12 +256,165 @@ The second command will fail and time out after 3 seconds. --- +## Network policies, pods, and services + +- Network policies apply to *pods* + +- A *service* can select multiple pods + + (And load balance traffic across them) + +- It is possible that we can connect to some pods, but not some others + + (Because of how network policies have been defined for these pods) + +- In that case, connections to the service will randomly pass or fail + + (Depending on whether the connection was sent to a pod that we have access to or not) + +--- + +## Network policies and namespaces + +- A good strategy is to isolate a namespace, so that: + + - all the pods in the namespace can communicate together + + - other namespaces cannot access the pods + + - external access has to be enabled explicitly + +- Let's see what this would look like for the DockerCoins app! + +--- + +## Network policies for DockerCoins + +- We are going to apply two policies + +- The first policy will prevent traffic from other namespaces + +- The second policy will allow traffic to the `webui` pods + +- That's all we need for that app! + +--- + +## Blocking traffic from other namespaces + +This policy selects all pods in the current namespace. + +It allows traffic only from pods in the current namespace. + +(An empty `podSelector` means "all pods".) + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: deny-from-other-namespaces +spec: + podSelector: {} + ingress: + - from: + - podSelector: {} +``` + +--- + +## Allowing traffic to `webui` pods + +This policy selects all pods with label `run=webui`. + +It allows traffic from any source. + +(An empty `from` fields means "all sources".) + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-webui +spec: + podSelector: + matchLabels: + run: webui + ingress: + - from: [] +``` + +--- + +## Applying both network policies + +- Both network policies are declared in the file `k8s/netpol-dockercoins.yaml` + +.exercise[ + +- Apply the network policies: + ```bash + kubectl apply -f ~/container.training/k8s/netpol-dockercoins.yaml + ``` + +- Check that we can still access the web UI from outside +
+ (and that the app is still working correctly!) + +- Check that we can't connect anymore to `rng` or `hasher` through their ClusterIP + +] + +Note: using `kubectl proxy` or `kubectl port-forward` allows to connect +regardless of existing network policies. This allows us to debug and +troubleshoot easily, without having to poke holes at our firewall. + +--- + +## Protecting the control plane + +- Should we add network policies to block unauthorized access to the control plane? + + (etcd, API server, etc.) + +-- + +- At first, it seems like a good idea ... + +-- + +- But it *shouldn't* be necessary: + + - not all network plugins support network policies + + - the control plane is secured by other methods (mutual TLS, mostly) + + - the code running in our pods can reasonably expect to contact the API +
+ (and it can do so safely thanks to the API permission model) + +- If we block access to the control plane, we might disrupt legitimate code + +- ... Without necessarily improving security + +--- + ## Further resources - As always, the [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/) is a good starting point +- The API documentation has a lot of detail about the format of various objects: + + - [NetworkPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#networkpolicy-v1-networking-k8s-io) + + - [NetworkPolicySpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#networkpolicyspec-v1-networking-k8s-io) + + - [NetworkPolicyIngressRule](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#networkpolicyingressrule-v1-networking-k8s-io) + + - etc. + - And two resources by [Ahmet Alp Balkan](https://ahmet.im/): - a [very good talk about network policies](https://www.youtube.com/watch?list=PLj6h78yzYM2P-3-xqvmWaZbbI1sW-ulZb&v=3gGpMmYeEO8) at KubeCon North America 2017 - a repository of [ready-to-use recipes](https://github.com/ahmetb/kubernetes-network-policy-recipes) for network policies + From 1e0954d9b4571fc6032ab88a012c1aaa6a2d6417 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Sat, 8 Sep 2018 08:49:37 -0500 Subject: [PATCH 6/6] Update netpol.md slight corrections --- slides/k8s/netpol.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slides/k8s/netpol.md b/slides/k8s/netpol.md index 6933b703..bb05b8a0 100644 --- a/slides/k8s/netpol.md +++ b/slides/k8s/netpol.md @@ -365,9 +365,9 @@ spec: ] -Note: using `kubectl proxy` or `kubectl port-forward` allows to connect +Note: using `kubectl proxy` or `kubectl port-forward` allows us to connect regardless of existing network policies. This allows us to debug and -troubleshoot easily, without having to poke holes at our firewall. +troubleshoot easily, without having to poke holes in our firewall. ---