github/capsule
1
0
Fork 0
mirror of https://github.com/projectcapsule/capsule.git synced 2026-08-19 20:46:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enforcing Service external IPs (#161)

Browse Source
This commit is contained in:
Dario Tranchitella
2020-12-11 19:17:46 +01:00
committed by GitHub
parent 007bdff512
commit 98e441f1e9
12 changed files with 460 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/v1alpha1/tenant_types.go
+8
View File
@@ -43,6 +43,13 @@ type ContainerRegistriesSpec struct {
AllowedRegex string `json:"allowedRegex,omitempty"`
}
// +kubebuilder:validation:Pattern="^([0-9]{1,3}.){3}[0-9]{1,3}(/([0-9]|[1-2][0-9]|3[0-2]))?$"
type AllowedIp string
type ExternalServiceIPs struct {
Allowed []AllowedIp `json:"allowed"`
}
// TenantSpec defines the desired state of Tenant
type TenantSpec struct {
Owner OwnerSpec `json:"owner"`
@@ -59,6 +66,7 @@ type TenantSpec struct {
LimitRanges []corev1.LimitRangeSpec `json:"limitRanges,omitempty"`
ResourceQuota []corev1.ResourceQuotaSpec `json:"resourceQuotas,omitempty"`
AdditionalRoleBindings []AdditionalRoleBindings `json:"additionalRoleBindings,omitempty"`
ExternalServiceIPs *ExternalServiceIPs `json:"externalServiceIPs,omitempty"`
}
type AdditionalRoleBindings struct {
api/v1alpha1/zz_generated.deepcopy.go
+25
View File
@@ -96,6 +96,26 @@ func (in *ContainerRegistriesSpec) DeepCopy() *ContainerRegistriesSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ExternalServiceIPs) DeepCopyInto(out *ExternalServiceIPs) {
*out = *in
if in.Allowed != nil {
in, out := &in.Allowed, &out.Allowed
*out = make([]AllowedIp, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalServiceIPs.
func (in *ExternalServiceIPs) DeepCopy() *ExternalServiceIPs {
if in == nil {
return nil
}
out := new(ExternalServiceIPs)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in IngressClassList) DeepCopyInto(out *IngressClassList) {
{
@@ -347,6 +367,11 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.ExternalServiceIPs != nil {
in, out := &in.ExternalServiceIPs, &out.ExternalServiceIPs
*out = new(ExternalServiceIPs)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantSpec.
charts/capsule/crds/tenant-crd.yaml
+9 -21
View File
@@ -112,9 +112,16 @@ spec:
type: array
allowedRegex:
type: string
type: object
externalServiceIPs:
properties:
allowed:
items:
pattern: ^([0-9]{1,3}.){3}[0-9]{1,3}(/([0-9]|[1-2][0-9]|3[0-2]))?$
type: string
type: array
required:
- allowed
- allowedRegex
type: object
ingressClasses:
properties:
@@ -124,9 +131,6 @@ spec:
type: array
allowedRegex:
type: string
required:
- allowed
- allowedRegex
type: object
limitRanges:
items:
@@ -207,6 +211,7 @@ spec:
type: array
namespaceQuota:
format: int32
minimum: 1
type: integer
namespacesMetadata:
properties:
@@ -218,9 +223,6 @@ spec:
additionalProperties:
type: string
type: object
required:
- additionalAnnotations
- additionalLabels
type: object
networkPolicies:
items:
@@ -772,9 +774,6 @@ spec:
additionalProperties:
type: string
type: object
required:
- additionalAnnotations
- additionalLabels
type: object
storageClasses:
properties:
@@ -784,9 +783,6 @@ spec:
type: array
allowedRegex:
type: string
required:
- allowed
- allowedRegex
type: object
required:
- owner
@@ -794,20 +790,12 @@ spec:
status:
description: TenantStatus defines the observed state of Tenant
properties:
groups:
items:
type: string
type: array
namespaces:
items:
type: string
type: array
size:
type: integer
users:
items:
type: string
type: array
required:
- size
type: object
charts/capsule/templates/validatingwebhookconfiguration.yaml
+30
View File
@@ -233,3 +233,33 @@ webhooks:
scope: '*'
sideEffects: NoneOnDryRun
timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }}
- admissionReviewVersions:
- v1beta1
clientConfig:
caBundle: Cg==
service:
name: {{ include "capsule.fullname" . }}-webhook-service
namespace: {{ .Release.Namespace }}
path: /validating-external-service-ips
port: 443
failurePolicy: Fail
matchPolicy: Exact
name: validating-external-service-ips.capsule.clastix.io
namespaceSelector:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
objectSelector: {}
rules:
- apiGroups:
- ""
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- services
scope: '*'
sideEffects: NoneOnDryRun
timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }}
config/crd/bases/capsule.clastix.io_tenants.yaml
+10
View File
@@ -115,6 +115,16 @@ spec:
allowedRegex:
type: string
type: object
externalServiceIPs:
properties:
allowed:
items:
pattern: ^([0-9]{1,3}.){3}[0-9]{1,3}(/([0-9]|[1-2][0-9]|3[0-2]))?$
type: string
type: array
required:
- allowed
type: object
ingressClasses:
properties:
allowed:
config/webhook/manifests.yaml
+18
View File
@@ -138,6 +138,24 @@ webhooks:
- CREATE
resources:
- pods
- clientConfig:
caBundle: Cg==
service:
name: webhook-service
namespace: system
path: /validating-external-service-ips
failurePolicy: Fail
name: validating-external-service-ips.capsule.clastix.io
rules:
- apiGroups:
- ""
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- services
- clientConfig:
caBundle: Cg==
service:
config/webhook/patch_ns_selector.yaml
+6
View File
@@ -28,3 +28,9 @@
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
- op: add
path: /webhooks/6/namespaceSelector
value:
matchExpressions:
- key: capsule.clastix.io/tenant
operator: Exists
e2e/allowed_external_ips_test.go
+155
View File
@@ -0,0 +1,155 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("enforcing an allowed set of Service External IPs", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "allowed-external-ip",
},
Spec: v1alpha1.TenantSpec{
Owner: v1alpha1.OwnerSpec{
Name: "google",
Kind: "User",
},
ExternalServiceIPs: &v1alpha1.ExternalServiceIPs{
Allowed: []v1alpha1.AllowedIp{
"10.20.0.0/16",
"192.168.1.2/32",
},
},
},
}
JustBeforeEach(func() {
EventuallyCreation(func() error {
return k8sClient.Create(context.TODO(), tnt.DeepCopy())
}).Should(Succeed())
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt.DeepCopy())).Should(Succeed())
})
It("should fail creating an evil service", func() {
ns := NewNamespace("evil-service")
NamespaceCreation(ns, tnt, defaultTimeoutInterval).Should(Succeed())
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "my-evil-dns-server",
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "dns",
Protocol: "UDP",
Port: 53,
TargetPort: intstr.FromInt(9053),
},
},
Selector: map[string]string{
"app": "my-evil-dns-server",
},
ExternalIPs: []string{
"8.8.8.8",
"8.8.4.4",
},
},
}
EventuallyCreation(func() error {
cs := ownerClient(tnt)
_, err := cs.CoreV1().Services(ns.Name).Create(context.Background(), svc, metav1.CreateOptions{})
return err
}).ShouldNot(Succeed())
})
It("should allow the first CIDR block", func() {
ns := NewNamespace("allowed-service-cidr")
NamespaceCreation(ns, tnt, defaultTimeoutInterval).Should(Succeed())
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "dns-server",
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "dns",
Protocol: "UDP",
Port: 53,
TargetPort: intstr.FromInt(9053),
},
},
Selector: map[string]string{
"app": "dns-server",
},
ExternalIPs: []string{
"10.20.0.0",
"10.20.255.255",
},
},
}
EventuallyCreation(func() error {
cs := ownerClient(tnt)
_, err := cs.CoreV1().Services(ns.Name).Create(context.Background(), svc, metav1.CreateOptions{})
return err
}).Should(Succeed())
})
It("should allow the /32 CIDR block", func() {
ns := NewNamespace("allowed-service-strict")
NamespaceCreation(ns, tnt, defaultTimeoutInterval).Should(Succeed())
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "dns-server",
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "dns",
Protocol: "UDP",
Port: 53,
TargetPort: intstr.FromInt(9053),
},
},
Selector: map[string]string{
"app": "dns-server",
},
ExternalIPs: []string{
"192.168.1.2",
},
},
}
EventuallyCreation(func() error {
cs := ownerClient(tnt)
_, err := cs.CoreV1().Services(ns.Name).Create(context.Background(), svc, metav1.CreateOptions{})
return err
}).Should(Succeed())
})
})
main.go
+2
View File
@@ -45,6 +45,7 @@ import (
"github.com/clastix/capsule/pkg/webhook/owner_reference"
"github.com/clastix/capsule/pkg/webhook/pvc"
"github.com/clastix/capsule/pkg/webhook/registry"
"github.com/clastix/capsule/pkg/webhook/services"
"github.com/clastix/capsule/pkg/webhook/tenant"
"github.com/clastix/capsule/pkg/webhook/tenant_prefix"
"github.com/clastix/capsule/pkg/webhook/utils"
@@ -159,6 +160,7 @@ func main() {
ingress.Webhook(ingress.Handler()),
pvc.Webhook(pvc.Handler()),
registry.Webhook(registry.Handler()),
services.Webhook(services.Handler()),
owner_reference.Webhook(utils.InCapsuleGroup(capsuleGroup, owner_reference.Handler(forceTenantPrefix))),
namespace_quota.Webhook(utils.InCapsuleGroup(capsuleGroup, namespace_quota.Handler())),
network_policies.Webhook(utils.InCapsuleGroup(capsuleGroup, network_policies.Handler())),
pkg/webhook/services/errors.go
+43
View File
@@ -0,0 +1,43 @@
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package services
import (
"fmt"
"strings"
"github.com/clastix/capsule/api/v1alpha1"
)
type externalServiceIPForbidden struct {
cidr []string
}
func NewExternalServiceIPForbidden(allowedIps []v1alpha1.AllowedIp) error {
var cidr []string
for _, i := range allowedIps {
cidr = append(cidr, string(i))
}
return &externalServiceIPForbidden{
cidr: cidr,
}
}
func (e externalServiceIPForbidden) Error() string {
return fmt.Sprintf("The selected external IPs for the current Service are violating the following enforced CIDRs: %s", strings.Join(e.cidr, ", "))
}
pkg/webhook/services/validating.go
+115
View File
@@ -0,0 +1,115 @@
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package services
import (
"context"
"net"
"net/http"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/fields"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
"github.com/clastix/capsule/api/v1alpha1"
capsulewebhook "github.com/clastix/capsule/pkg/webhook"
)
// +kubebuilder:webhook:path=/validating-external-service-ips,mutating=false,failurePolicy=fail,groups="",resources=services,verbs=create;update,versions=v1,name=validating-external-service-ips.capsule.clastix.io
type webhook struct {
handler capsulewebhook.Handler
}
func Webhook(handler capsulewebhook.Handler) capsulewebhook.Webhook {
return &webhook{handler: handler}
}
func (w *webhook) GetHandler() capsulewebhook.Handler {
return w.handler
}
func (w *webhook) GetName() string {
return "Service"
}
func (w *webhook) GetPath() string {
return "/validating-external-service-ips"
}
type handler struct{}
func Handler() capsulewebhook.Handler {
return &handler{}
}
func (r *handler) handleService(clt client.Client, decoder *admission.Decoder, ctx context.Context, req admission.Request) admission.Response {
s := &corev1.Service{}
if err := decoder.Decode(req, s); err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
if s.Spec.ExternalIPs == nil {
return admission.Allowed("")
}
tl := &v1alpha1.TenantList{}
if err := clt.List(ctx, tl, client.MatchingFieldsSelector{
Selector: fields.OneTermEqualSelector(".status.namespaces", s.GetNamespace()),
}); err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
if len(tl.Items) == 0 {
return admission.Allowed("")
}
tnt := tl.Items[0]
if tnt.Spec.ExternalServiceIPs == nil {
return admission.Allowed("")
}
for _, allowed := range tnt.Spec.ExternalServiceIPs.Allowed {
_, allowedIp, _ := net.ParseCIDR(string(allowed))
for _, externalIp := range s.Spec.ExternalIPs {
IP := net.ParseIP(externalIp)
if allowedIp.Contains(IP) {
return admission.Allowed("")
}
}
}
return admission.Errored(http.StatusBadRequest, NewExternalServiceIPForbidden(tnt.Spec.ExternalServiceIPs.Allowed))
}
func (r *handler) OnCreate(client client.Client, decoder *admission.Decoder) capsulewebhook.Func {
return func(ctx context.Context, req admission.Request) admission.Response {
return r.handleService(client, decoder, ctx, req)
}
}
func (r *handler) OnUpdate(client client.Client, decoder *admission.Decoder) capsulewebhook.Func {
return func(ctx context.Context, req admission.Request) admission.Response {
return r.handleService(client, decoder, ctx, req)
}
}
func (r *handler) OnDelete(client client.Client, decoder *admission.Decoder) capsulewebhook.Func {
return func(ctx context.Context, req admission.Request) admission.Response {
return admission.Allowed("")
}
}
use_cases.md
+39 -1
View File
@@ -1027,4 +1027,42 @@ Labels: capsule.clastix.io/tenant=oil
Annotations: capsule.clastix.io/allowed-registries: docker.io
capsule.clastix.io/allowed-registries-regexp: ^registry\.internal\.\w+$
...
```
```
# Mitigating CVE-2020-8554 (Man in the middle using LoadBalancer or ExternalIPs)
__Capsule__ is able to enforce the external IPs an end-user would try to assign
to a Service, mitigating the [CVE-2020-8554](https://github.com/kubernetes/kubernetes/issues/97076).
```yaml
apiVersion: capsule.clastix.io/v1alpha1
kind: Tenant
spec:
externalServiceIPs:
allowed:
- 10.0.0.1/32
```
Trying to create a __Service__ using an IP address non contained in the
specified ranges will fail.
```
alice@caas# cat <<EOF | kubectl apply -f
apiVersion: v1
kind: Service
metadata:
name: my-evil-service
spec:
selector:
app: my-evil-dns-server
ports:
- name: dns
protocol: UDP
port: 53
targetPort: 9053
externalIPs:
- 8.8.8.8
- 8.8.4.4
EOF
Error from server: error when creating "STDIN": admission webhook "validating-external-service-ips.capsule.clastix.io" denied the request: The External IP 8.8.8.8 is forbidden for the current Tenant, violating the CIDR 10.0.0.1/32
```