diff --git a/api/v1alpha1/tenant_types.go b/api/v1alpha1/tenant_types.go index efe197d2..fcb3d0b4 100644 --- a/api/v1alpha1/tenant_types.go +++ b/api/v1alpha1/tenant_types.go @@ -43,6 +43,13 @@ type ContainerRegistriesSpec struct { AllowedRegex string `json:"allowedRegex,omitempty"` } +// +kubebuilder:validation:Pattern="^([0-9]{1,3}.){3}[0-9]{1,3}(/([0-9]|[1-2][0-9]|3[0-2]))?$" +type AllowedIp string + +type ExternalServiceIPs struct { + Allowed []AllowedIp `json:"allowed"` +} + // TenantSpec defines the desired state of Tenant type TenantSpec struct { Owner OwnerSpec `json:"owner"` @@ -59,6 +66,7 @@ type TenantSpec struct { LimitRanges []corev1.LimitRangeSpec `json:"limitRanges,omitempty"` ResourceQuota []corev1.ResourceQuotaSpec `json:"resourceQuotas,omitempty"` AdditionalRoleBindings []AdditionalRoleBindings `json:"additionalRoleBindings,omitempty"` + ExternalServiceIPs *ExternalServiceIPs `json:"externalServiceIPs,omitempty"` } type AdditionalRoleBindings struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e5a1c530..e1f41175 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -96,6 +96,26 @@ func (in *ContainerRegistriesSpec) DeepCopy() *ContainerRegistriesSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalServiceIPs) DeepCopyInto(out *ExternalServiceIPs) { + *out = *in + if in.Allowed != nil { + in, out := &in.Allowed, &out.Allowed + *out = make([]AllowedIp, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalServiceIPs. +func (in *ExternalServiceIPs) DeepCopy() *ExternalServiceIPs { + if in == nil { + return nil + } + out := new(ExternalServiceIPs) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in IngressClassList) DeepCopyInto(out *IngressClassList) { { @@ -347,6 +367,11 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ExternalServiceIPs != nil { + in, out := &in.ExternalServiceIPs, &out.ExternalServiceIPs + *out = new(ExternalServiceIPs) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TenantSpec. diff --git a/charts/capsule/crds/tenant-crd.yaml b/charts/capsule/crds/tenant-crd.yaml index 105f0a14..2b65c702 100644 --- a/charts/capsule/crds/tenant-crd.yaml +++ b/charts/capsule/crds/tenant-crd.yaml @@ -112,9 +112,16 @@ spec: type: array allowedRegex: type: string + type: object + externalServiceIPs: + properties: + allowed: + items: + pattern: ^([0-9]{1,3}.){3}[0-9]{1,3}(/([0-9]|[1-2][0-9]|3[0-2]))?$ + type: string + type: array required: - allowed - - allowedRegex type: object ingressClasses: properties: @@ -124,9 +131,6 @@ spec: type: array allowedRegex: type: string - required: - - allowed - - allowedRegex type: object limitRanges: items: @@ -207,6 +211,7 @@ spec: type: array namespaceQuota: format: int32 + minimum: 1 type: integer namespacesMetadata: properties: @@ -218,9 +223,6 @@ spec: additionalProperties: type: string type: object - required: - - additionalAnnotations - - additionalLabels type: object networkPolicies: items: @@ -772,9 +774,6 @@ spec: additionalProperties: type: string type: object - required: - - additionalAnnotations - - additionalLabels type: object storageClasses: properties: @@ -784,9 +783,6 @@ spec: type: array allowedRegex: type: string - required: - - allowed - - allowedRegex type: object required: - owner @@ -794,20 +790,12 @@ spec: status: description: TenantStatus defines the observed state of Tenant properties: - groups: - items: - type: string - type: array namespaces: items: type: string type: array size: type: integer - users: - items: - type: string - type: array required: - size type: object diff --git a/charts/capsule/templates/validatingwebhookconfiguration.yaml b/charts/capsule/templates/validatingwebhookconfiguration.yaml index d9bc0051..d77fcedd 100644 --- a/charts/capsule/templates/validatingwebhookconfiguration.yaml +++ b/charts/capsule/templates/validatingwebhookconfiguration.yaml @@ -233,3 +233,33 @@ webhooks: scope: '*' sideEffects: NoneOnDryRun timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }} +- admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: {{ include "capsule.fullname" . }}-webhook-service + namespace: {{ .Release.Namespace }} + path: /validating-external-service-ips + port: 443 + failurePolicy: Fail + matchPolicy: Exact + name: validating-external-service-ips.capsule.clastix.io + namespaceSelector: + matchExpressions: + - key: capsule.clastix.io/tenant + operator: Exists + objectSelector: {} + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - services + scope: '*' + sideEffects: NoneOnDryRun + timeoutSeconds: {{ .Values.validatingWebhooksTimeoutSeconds }} diff --git a/config/crd/bases/capsule.clastix.io_tenants.yaml b/config/crd/bases/capsule.clastix.io_tenants.yaml index 93a20fa9..b8e1a5d7 100644 --- a/config/crd/bases/capsule.clastix.io_tenants.yaml +++ b/config/crd/bases/capsule.clastix.io_tenants.yaml @@ -115,6 +115,16 @@ spec: allowedRegex: type: string type: object + externalServiceIPs: + properties: + allowed: + items: + pattern: ^([0-9]{1,3}.){3}[0-9]{1,3}(/([0-9]|[1-2][0-9]|3[0-2]))?$ + type: string + type: array + required: + - allowed + type: object ingressClasses: properties: allowed: diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index bbbbc4f4..27ef998e 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -138,6 +138,24 @@ webhooks: - CREATE resources: - pods +- clientConfig: + caBundle: Cg== + service: + name: webhook-service + namespace: system + path: /validating-external-service-ips + failurePolicy: Fail + name: validating-external-service-ips.capsule.clastix.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - services - clientConfig: caBundle: Cg== service: diff --git a/config/webhook/patch_ns_selector.yaml b/config/webhook/patch_ns_selector.yaml index abd90125..7f6b53e1 100644 --- a/config/webhook/patch_ns_selector.yaml +++ b/config/webhook/patch_ns_selector.yaml @@ -28,3 +28,9 @@ matchExpressions: - key: capsule.clastix.io/tenant operator: Exists +- op: add + path: /webhooks/6/namespaceSelector + value: + matchExpressions: + - key: capsule.clastix.io/tenant + operator: Exists diff --git a/e2e/allowed_external_ips_test.go b/e2e/allowed_external_ips_test.go new file mode 100644 index 00000000..e878219c --- /dev/null +++ b/e2e/allowed_external_ips_test.go @@ -0,0 +1,155 @@ +//+build e2e + +/* +Copyright 2020 Clastix Labs. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/clastix/capsule/api/v1alpha1" +) + +var _ = Describe("enforcing an allowed set of Service External IPs", func() { + tnt := &v1alpha1.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "allowed-external-ip", + }, + Spec: v1alpha1.TenantSpec{ + Owner: v1alpha1.OwnerSpec{ + Name: "google", + Kind: "User", + }, + ExternalServiceIPs: &v1alpha1.ExternalServiceIPs{ + Allowed: []v1alpha1.AllowedIp{ + "10.20.0.0/16", + "192.168.1.2/32", + }, + }, + }, + } + JustBeforeEach(func() { + EventuallyCreation(func() error { + return k8sClient.Create(context.TODO(), tnt.DeepCopy()) + }).Should(Succeed()) + }) + JustAfterEach(func() { + Expect(k8sClient.Delete(context.TODO(), tnt.DeepCopy())).Should(Succeed()) + }) + It("should fail creating an evil service", func() { + ns := NewNamespace("evil-service") + NamespaceCreation(ns, tnt, defaultTimeoutInterval).Should(Succeed()) + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-evil-dns-server", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "dns", + Protocol: "UDP", + Port: 53, + TargetPort: intstr.FromInt(9053), + }, + }, + Selector: map[string]string{ + "app": "my-evil-dns-server", + }, + ExternalIPs: []string{ + "8.8.8.8", + "8.8.4.4", + }, + }, + } + EventuallyCreation(func() error { + cs := ownerClient(tnt) + _, err := cs.CoreV1().Services(ns.Name).Create(context.Background(), svc, metav1.CreateOptions{}) + return err + }).ShouldNot(Succeed()) + }) + It("should allow the first CIDR block", func() { + ns := NewNamespace("allowed-service-cidr") + NamespaceCreation(ns, tnt, defaultTimeoutInterval).Should(Succeed()) + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dns-server", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "dns", + Protocol: "UDP", + Port: 53, + TargetPort: intstr.FromInt(9053), + }, + }, + Selector: map[string]string{ + "app": "dns-server", + }, + ExternalIPs: []string{ + "10.20.0.0", + "10.20.255.255", + }, + }, + } + EventuallyCreation(func() error { + cs := ownerClient(tnt) + _, err := cs.CoreV1().Services(ns.Name).Create(context.Background(), svc, metav1.CreateOptions{}) + return err + }).Should(Succeed()) + }) + + It("should allow the /32 CIDR block", func() { + ns := NewNamespace("allowed-service-strict") + NamespaceCreation(ns, tnt, defaultTimeoutInterval).Should(Succeed()) + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dns-server", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "dns", + Protocol: "UDP", + Port: 53, + TargetPort: intstr.FromInt(9053), + }, + }, + Selector: map[string]string{ + "app": "dns-server", + }, + ExternalIPs: []string{ + "192.168.1.2", + }, + }, + } + EventuallyCreation(func() error { + cs := ownerClient(tnt) + _, err := cs.CoreV1().Services(ns.Name).Create(context.Background(), svc, metav1.CreateOptions{}) + return err + }).Should(Succeed()) + }) +}) diff --git a/main.go b/main.go index b2958684..301d89a9 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ import ( "github.com/clastix/capsule/pkg/webhook/owner_reference" "github.com/clastix/capsule/pkg/webhook/pvc" "github.com/clastix/capsule/pkg/webhook/registry" + "github.com/clastix/capsule/pkg/webhook/services" "github.com/clastix/capsule/pkg/webhook/tenant" "github.com/clastix/capsule/pkg/webhook/tenant_prefix" "github.com/clastix/capsule/pkg/webhook/utils" @@ -159,6 +160,7 @@ func main() { ingress.Webhook(ingress.Handler()), pvc.Webhook(pvc.Handler()), registry.Webhook(registry.Handler()), + services.Webhook(services.Handler()), owner_reference.Webhook(utils.InCapsuleGroup(capsuleGroup, owner_reference.Handler(forceTenantPrefix))), namespace_quota.Webhook(utils.InCapsuleGroup(capsuleGroup, namespace_quota.Handler())), network_policies.Webhook(utils.InCapsuleGroup(capsuleGroup, network_policies.Handler())), diff --git a/pkg/webhook/services/errors.go b/pkg/webhook/services/errors.go new file mode 100644 index 00000000..42470780 --- /dev/null +++ b/pkg/webhook/services/errors.go @@ -0,0 +1,43 @@ +/* +Copyright 2020 Clastix Labs. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "fmt" + "strings" + + "github.com/clastix/capsule/api/v1alpha1" +) + +type externalServiceIPForbidden struct { + cidr []string +} + +func NewExternalServiceIPForbidden(allowedIps []v1alpha1.AllowedIp) error { + var cidr []string + for _, i := range allowedIps { + cidr = append(cidr, string(i)) + } + return &externalServiceIPForbidden{ + cidr: cidr, + } +} + +func (e externalServiceIPForbidden) Error() string { + + return fmt.Sprintf("The selected external IPs for the current Service are violating the following enforced CIDRs: %s", strings.Join(e.cidr, ", ")) +} diff --git a/pkg/webhook/services/validating.go b/pkg/webhook/services/validating.go new file mode 100644 index 00000000..eac05e5f --- /dev/null +++ b/pkg/webhook/services/validating.go @@ -0,0 +1,115 @@ +/* +Copyright 2020 Clastix Labs. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "context" + "net" + "net/http" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/clastix/capsule/api/v1alpha1" + capsulewebhook "github.com/clastix/capsule/pkg/webhook" +) + +// +kubebuilder:webhook:path=/validating-external-service-ips,mutating=false,failurePolicy=fail,groups="",resources=services,verbs=create;update,versions=v1,name=validating-external-service-ips.capsule.clastix.io + +type webhook struct { + handler capsulewebhook.Handler +} + +func Webhook(handler capsulewebhook.Handler) capsulewebhook.Webhook { + return &webhook{handler: handler} +} + +func (w *webhook) GetHandler() capsulewebhook.Handler { + return w.handler +} + +func (w *webhook) GetName() string { + return "Service" +} + +func (w *webhook) GetPath() string { + return "/validating-external-service-ips" +} + +type handler struct{} + +func Handler() capsulewebhook.Handler { + return &handler{} +} + +func (r *handler) handleService(clt client.Client, decoder *admission.Decoder, ctx context.Context, req admission.Request) admission.Response { + s := &corev1.Service{} + if err := decoder.Decode(req, s); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if s.Spec.ExternalIPs == nil { + return admission.Allowed("") + } + + tl := &v1alpha1.TenantList{} + if err := clt.List(ctx, tl, client.MatchingFieldsSelector{ + Selector: fields.OneTermEqualSelector(".status.namespaces", s.GetNamespace()), + }); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + if len(tl.Items) == 0 { + return admission.Allowed("") + } + tnt := tl.Items[0] + + if tnt.Spec.ExternalServiceIPs == nil { + return admission.Allowed("") + } + + for _, allowed := range tnt.Spec.ExternalServiceIPs.Allowed { + _, allowedIp, _ := net.ParseCIDR(string(allowed)) + for _, externalIp := range s.Spec.ExternalIPs { + IP := net.ParseIP(externalIp) + if allowedIp.Contains(IP) { + return admission.Allowed("") + } + } + } + + return admission.Errored(http.StatusBadRequest, NewExternalServiceIPForbidden(tnt.Spec.ExternalServiceIPs.Allowed)) +} + +func (r *handler) OnCreate(client client.Client, decoder *admission.Decoder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) admission.Response { + return r.handleService(client, decoder, ctx, req) + } +} + +func (r *handler) OnUpdate(client client.Client, decoder *admission.Decoder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) admission.Response { + return r.handleService(client, decoder, ctx, req) + } +} + +func (r *handler) OnDelete(client client.Client, decoder *admission.Decoder) capsulewebhook.Func { + return func(ctx context.Context, req admission.Request) admission.Response { + return admission.Allowed("") + } +} diff --git a/use_cases.md b/use_cases.md index 881667e4..222a41d6 100644 --- a/use_cases.md +++ b/use_cases.md @@ -1027,4 +1027,42 @@ Labels: capsule.clastix.io/tenant=oil Annotations: capsule.clastix.io/allowed-registries: docker.io capsule.clastix.io/allowed-registries-regexp: ^registry\.internal\.\w+$ ... -``` \ No newline at end of file +``` + +# Mitigating CVE-2020-8554 (Man in the middle using LoadBalancer or ExternalIPs) + +__Capsule__ is able to enforce the external IPs an end-user would try to assign +to a Service, mitigating the [CVE-2020-8554](https://github.com/kubernetes/kubernetes/issues/97076). + +```yaml +apiVersion: capsule.clastix.io/v1alpha1 +kind: Tenant +spec: + externalServiceIPs: + allowed: + - 10.0.0.1/32 +``` + +Trying to create a __Service__ using an IP address non contained in the +specified ranges will fail. + +``` +alice@caas# cat <