feat(api): label selector for storage, ingress, podpriority classes

This commit is contained in:
Dario Tranchitella
2022-12-23 15:31:59 +01:00
parent 289b079530
commit 93fbca9b18
13 changed files with 232 additions and 42 deletions

View File

@@ -61,9 +61,11 @@ func generateTenantsSpecs() (Tenant, capsulev1beta1.Tenant) {
Allowed: []api.AllowedIP{"192.168.0.1"},
},
}
v1beta1AllowedListSpec := &api.AllowedListSpec{
Exact: []string{"foo", "bar"},
Regex: "^foo*",
v1beta2AllowedListSpec := &api.SelectorAllowedListSpec{
AllowedListSpec: api.AllowedListSpec{
Exact: []string{"foo", "bar"},
Regex: "^foo*",
},
}
networkPolicies := []networkingv1.NetworkPolicySpec{
{
@@ -235,13 +237,13 @@ func generateTenantsSpecs() (Tenant, capsulev1beta1.Tenant) {
},
NamespaceOptions: v1beta1NamespaceOptions,
ServiceOptions: v1beta1ServiceOptions,
StorageClasses: v1beta1AllowedListSpec,
StorageClasses: &v1beta2AllowedListSpec.AllowedListSpec,
IngressOptions: capsulev1beta1.IngressOptions{
HostnameCollisionScope: api.HostnameCollisionScopeDisabled,
AllowedClasses: v1beta1AllowedListSpec,
AllowedHostnames: v1beta1AllowedListSpec,
AllowedClasses: &v1beta2AllowedListSpec.AllowedListSpec,
AllowedHostnames: &v1beta2AllowedListSpec.AllowedListSpec,
},
ContainerRegistries: v1beta1AllowedListSpec,
ContainerRegistries: &v1beta2AllowedListSpec.AllowedListSpec,
NodeSelector: nodeSelector,
NetworkPolicies: api.NetworkPolicySpec{
Items: networkPolicies,

View File

@@ -9,7 +9,7 @@ import (
type IngressOptions struct {
// Specifies the allowed IngressClasses assigned to the Tenant. Capsule assures that all Ingress resources created in the Tenant can use only one of the allowed IngressClasses. Optional.
AllowedClasses *api.AllowedListSpec `json:"allowedClasses,omitempty"`
AllowedClasses *api.SelectorAllowedListSpec `json:"allowedClasses,omitempty"`
// Defines the scope of hostname collision check performed when Tenant Owners create Ingress with allowed hostnames.
//
//

View File

@@ -85,7 +85,11 @@ func (in *Tenant) ConvertFrom(raw conversion.Hub) error {
}
in.Spec.ServiceOptions = src.Spec.ServiceOptions
in.Spec.StorageClasses = src.Spec.StorageClasses
if src.Spec.StorageClasses != nil {
in.Spec.StorageClasses = &api.SelectorAllowedListSpec{
AllowedListSpec: *src.Spec.StorageClasses,
}
}
if scope := src.Spec.IngressOptions.HostnameCollisionScope; len(scope) > 0 {
in.Spec.IngressOptions.HostnameCollisionScope = scope
@@ -102,7 +106,9 @@ func (in *Tenant) ConvertFrom(raw conversion.Hub) error {
}
if ingressClass := src.Spec.IngressOptions.AllowedClasses; ingressClass != nil {
in.Spec.IngressOptions.AllowedClasses = ingressClass
in.Spec.IngressOptions.AllowedClasses = &api.SelectorAllowedListSpec{
AllowedListSpec: *ingressClass,
}
}
if hostnames := src.Spec.IngressOptions.AllowedHostnames; hostnames != nil {
@@ -116,7 +122,12 @@ func (in *Tenant) ConvertFrom(raw conversion.Hub) error {
in.Spec.ResourceQuota = src.Spec.ResourceQuota
in.Spec.AdditionalRoleBindings = src.Spec.AdditionalRoleBindings
in.Spec.ImagePullPolicies = src.Spec.ImagePullPolicies
in.Spec.PriorityClasses = src.Spec.PriorityClasses
if src.Spec.PriorityClasses != nil {
in.Spec.PriorityClasses = &api.SelectorAllowedListSpec{
AllowedListSpec: *src.Spec.PriorityClasses,
}
}
if v, found := annotations["capsule.clastix.io/cordon"]; found {
value, err := strconv.ParseBool(v)
@@ -207,12 +218,14 @@ func (in *Tenant) ConvertTo(raw conversion.Hub) error {
}
dst.Spec.ServiceOptions = in.Spec.ServiceOptions
dst.Spec.StorageClasses = in.Spec.StorageClasses
if in.Spec.StorageClasses != nil {
dst.Spec.StorageClasses = &in.Spec.StorageClasses.AllowedListSpec
}
dst.Spec.IngressOptions.HostnameCollisionScope = in.Spec.IngressOptions.HostnameCollisionScope
if allowed := in.Spec.IngressOptions.AllowedClasses; allowed != nil {
dst.Spec.IngressOptions.AllowedClasses = allowed
dst.Spec.IngressOptions.AllowedClasses = &allowed.AllowedListSpec
}
if allowed := in.Spec.IngressOptions.AllowedHostnames; allowed != nil {
@@ -231,7 +244,10 @@ func (in *Tenant) ConvertTo(raw conversion.Hub) error {
dst.Spec.ResourceQuota = in.Spec.ResourceQuota
dst.Spec.AdditionalRoleBindings = in.Spec.AdditionalRoleBindings
dst.Spec.ImagePullPolicies = in.Spec.ImagePullPolicies
dst.Spec.PriorityClasses = in.Spec.PriorityClasses
if in.Spec.PriorityClasses != nil {
dst.Spec.PriorityClasses = &in.Spec.PriorityClasses.AllowedListSpec
}
if in.Spec.PreventDeletion {
annotations[api.ProtectedTenantAnnotation] = "true" //nolint:goconst

View File

@@ -18,7 +18,7 @@ type TenantSpec struct {
// Specifies options for the Service, such as additional metadata or block of certain type of Services. Optional.
ServiceOptions *api.ServiceOptions `json:"serviceOptions,omitempty"`
// Specifies the allowed StorageClasses assigned to the Tenant. Capsule assures that all PersistentVolumeClaim resources created in the Tenant can use only one of the allowed StorageClasses. Optional.
StorageClasses *api.AllowedListSpec `json:"storageClasses,omitempty"`
StorageClasses *api.SelectorAllowedListSpec `json:"storageClasses,omitempty"`
// Specifies options for the Ingress resources, such as allowed hostnames and IngressClass. Optional.
IngressOptions IngressOptions `json:"ingressOptions,omitempty"`
// Specifies the trusted Image Registries assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant can use only one of the allowed trusted registries. Optional.
@@ -36,7 +36,7 @@ type TenantSpec struct {
// Specify the allowed values for the imagePullPolicies option in Pod resources. Capsule assures that all Pod resources created in the Tenant can use only one of the allowed policy. Optional.
ImagePullPolicies []api.ImagePullPolicySpec `json:"imagePullPolicies,omitempty"`
// Specifies the allowed priorityClasses assigned to the Tenant. Capsule assures that all Pods resources created in the Tenant can use only one of the allowed PriorityClasses. Optional.
PriorityClasses *api.AllowedListSpec `json:"priorityClasses,omitempty"`
PriorityClasses *api.SelectorAllowedListSpec `json:"priorityClasses,omitempty"`
// Toggling the Tenant resources cordoning, when enable resources cannot be deleted.
Cordoned bool `json:"cordoned,omitempty"`
// Prevent accidental deletion of the Tenant.

View File

@@ -261,7 +261,7 @@ func (in *IngressOptions) DeepCopyInto(out *IngressOptions) {
*out = *in
if in.AllowedClasses != nil {
in, out := &in.AllowedClasses, &out.AllowedClasses
*out = new(api.AllowedListSpec)
*out = new(api.SelectorAllowedListSpec)
(*in).DeepCopyInto(*out)
}
if in.AllowedHostnames != nil {
@@ -718,7 +718,7 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) {
}
if in.StorageClasses != nil {
in, out := &in.StorageClasses, &out.StorageClasses
*out = new(api.AllowedListSpec)
*out = new(api.SelectorAllowedListSpec)
(*in).DeepCopyInto(*out)
}
in.IngressOptions.DeepCopyInto(&out.IngressOptions)
@@ -751,7 +751,7 @@ func (in *TenantSpec) DeepCopyInto(out *TenantSpec) {
}
if in.PriorityClasses != nil {
in, out := &in.PriorityClasses, &out.PriorityClasses
*out = new(api.AllowedListSpec)
*out = new(api.SelectorAllowedListSpec)
(*in).DeepCopyInto(*out)
}
}

View File

@@ -7,10 +7,31 @@ import (
"regexp"
"sort"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// +kubebuilder:object:generate=true
type SelectorAllowedListSpec struct {
AllowedListSpec `json:",inline"`
Selector metav1.LabelSelector `json:",inline"`
}
func (in *SelectorAllowedListSpec) SelectorMatch(obj client.Object) bool {
selector, err := metav1.LabelSelectorAsSelector(&in.Selector)
if err != nil {
return false
}
return selector.Matches(labels.Set(obj.GetLabels()))
}
// +kubebuilder:object:generate=true
type AllowedListSpec struct {
Exact []string `json:"allowed,omitempty"`
Regex string `json:"allowedRegex,omitempty"`

View File

@@ -219,6 +219,23 @@ func (in *ResourceQuotaSpec) DeepCopy() *ResourceQuotaSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SelectorAllowedListSpec) DeepCopyInto(out *SelectorAllowedListSpec) {
*out = *in
in.AllowedListSpec.DeepCopyInto(&out.AllowedListSpec)
in.Selector.DeepCopyInto(&out.Selector)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SelectorAllowedListSpec.
func (in *SelectorAllowedListSpec) DeepCopy() *SelectorAllowedListSpec {
if in == nil {
return nil
}
out := new(SelectorAllowedListSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ServiceOptions) DeepCopyInto(out *ServiceOptions) {
*out = *in

View File

@@ -12,10 +12,10 @@ import (
type ingressClassForbiddenError struct {
className string
spec api.AllowedListSpec
spec api.SelectorAllowedListSpec
}
func NewIngressClassForbidden(className string, spec api.AllowedListSpec) error {
func NewIngressClassForbidden(className string, spec api.SelectorAllowedListSpec) error {
return &ingressClassForbiddenError{
className: className,
spec: spec,
@@ -54,10 +54,10 @@ func (i ingressHostnameNotValidError) Error() string {
}
type ingressClassNotValidError struct {
spec api.AllowedListSpec
spec api.SelectorAllowedListSpec
}
func NewIngressClassNotValid(spec api.AllowedListSpec) error {
func NewIngressClassNotValid(spec api.SelectorAllowedListSpec) error {
return &ingressClassNotValidError{
spec: spec,
}
@@ -68,7 +68,7 @@ func (i ingressClassNotValidError) Error() string {
}
// nolint:predeclared
func appendClassError(spec api.AllowedListSpec) (append string) {
func appendClassError(spec api.SelectorAllowedListSpec) (append string) {
if len(spec.Exact) > 0 {
append += fmt.Sprintf(", one of the following (%s)", strings.Join(spec.Exact, ", "))
}
@@ -77,6 +77,10 @@ func appendClassError(spec api.AllowedListSpec) (append string) {
append += fmt.Sprintf(", or matching the regex %s", spec.Regex)
}
if len(spec.Selector.MatchLabels) > 0 || len(spec.Selector.MatchExpressions) > 0 {
append += fmt.Sprintf(", or matching the label selector defined in the Tenant")
}
return
}

View File

@@ -5,9 +5,15 @@ package ingress
import (
"context"
"net/http"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
networkingv1beta1 "k8s.io/api/networking/v1beta1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -20,10 +26,43 @@ import (
type class struct {
configuration configuration.Configuration
version *version.Version
}
func Class(configuration configuration.Configuration) capsulewebhook.Handler {
return &class{configuration: configuration}
version, _ := utils.GetK8sVersion()
return &class{
configuration: configuration,
version: version,
}
}
func (r *class) retrieveIngressClass(ctx context.Context, ctrlClient client.Client, ingressClassName *string) (client.Object, error) {
if r.version == nil || ingressClassName == nil {
return nil, nil
}
var obj client.Object
switch {
case r.version.Minor() < 18:
return nil, nil
case r.version.Minor() < 19:
obj = &networkingv1beta1.IngressClass{}
default:
obj = &networkingv1.IngressClass{}
}
if err := ctrlClient.Get(ctx, types.NamespacedName{Name: *ingressClassName}, obj); err != nil {
if k8serrors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
return obj, nil
}
// nolint:dupl
@@ -45,7 +84,14 @@ func (r *class) OnCreate(client client.Client, decoder *admission.Decoder, recor
return nil
}
if err = r.validateClass(*tenant, ingress.IngressClass()); err == nil {
ic, err := r.retrieveIngressClass(ctx, client, ingress.IngressClass())
if err != nil {
response := admission.Errored(http.StatusInternalServerError, err)
return &response
}
if err = r.validateClass(*tenant, ingress.IngressClass(), ic); err == nil {
return nil
}
@@ -86,7 +132,14 @@ func (r *class) OnUpdate(client client.Client, decoder *admission.Decoder, recor
return nil
}
if err = r.validateClass(*tenant, ingress.IngressClass()); err == nil {
ic, err := r.retrieveIngressClass(ctx, client, ingress.IngressClass())
if err != nil {
response := admission.Errored(http.StatusInternalServerError, err)
return &response
}
if err = r.validateClass(*tenant, ingress.IngressClass(), ic); err == nil {
return nil
}
@@ -114,7 +167,7 @@ func (r *class) OnDelete(client.Client, *admission.Decoder, record.EventRecorder
}
}
func (r *class) validateClass(tenant capsulev1beta2.Tenant, ingressClass *string) error {
func (r *class) validateClass(tenant capsulev1beta2.Tenant, ingressClass *string, ingressClassObj client.Object) error {
if tenant.Spec.IngressOptions.AllowedClasses == nil {
return nil
}
@@ -123,15 +176,21 @@ func (r *class) validateClass(tenant capsulev1beta2.Tenant, ingressClass *string
return NewIngressClassNotValid(*tenant.Spec.IngressOptions.AllowedClasses)
}
var valid, matched bool
var valid, regex, match bool
if len(tenant.Spec.IngressOptions.AllowedClasses.Exact) > 0 {
valid = tenant.Spec.IngressOptions.AllowedClasses.ExactMatch(*ingressClass)
}
matched = tenant.Spec.IngressOptions.AllowedClasses.RegexMatch(*ingressClass)
regex = tenant.Spec.IngressOptions.AllowedClasses.RegexMatch(*ingressClass)
if !valid && !matched {
if ingressClassObj != nil {
match = tenant.Spec.IngressOptions.AllowedClasses.SelectorMatch(ingressClassObj)
} else {
match = true
}
if !valid && !regex && !match {
return NewIngressClassForbidden(*ingressClass, *tenant.Spec.IngressOptions.AllowedClasses)
}

View File

@@ -5,9 +5,13 @@ package pod
import (
"context"
"net/http"
corev1 "k8s.io/api/core/v1"
schedulingv1 "k8s.io/api/scheduling/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -23,6 +27,24 @@ func PriorityClass() capsulewebhook.Handler {
return &priorityClass{}
}
func (h *priorityClass) class(ctx context.Context, c client.Client, name string) (client.Object, error) {
if len(name) == 0 {
return nil, nil
}
obj := &schedulingv1.PriorityClass{}
if err := c.Get(ctx, types.NamespacedName{Name: name}, obj); err != nil {
if errors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
return obj, nil
}
func (h *priorityClass) OnCreate(c client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
pod := &corev1.Pod{}
@@ -46,6 +68,13 @@ func (h *priorityClass) OnCreate(c client.Client, decoder *admission.Decoder, re
priorityClassName := pod.Spec.PriorityClassName
class, err := h.class(ctx, c, priorityClassName)
if err != nil {
response := admission.Errored(http.StatusInternalServerError, err)
return &response
}
switch {
case allowed == nil:
// Enforcement is not in place, skipping it at all
@@ -53,7 +82,7 @@ func (h *priorityClass) OnCreate(c client.Client, decoder *admission.Decoder, re
case len(priorityClassName) == 0:
// We don't have to force Pod to specify a Priority Class
return nil
case !allowed.ExactMatch(priorityClassName) && !allowed.RegexMatch(priorityClassName):
case !allowed.ExactMatch(priorityClassName) && !allowed.RegexMatch(priorityClassName) && !allowed.SelectorMatch(class):
recorder.Eventf(&tntList.Items[0], corev1.EventTypeWarning, "ForbiddenPriorityClass", "Pod %s/%s is using Priority Class %s is forbidden for the current Tenant", pod.Namespace, pod.Name, priorityClassName)
response := admission.Denied(NewPodPriorityClassForbidden(priorityClassName, *allowed).Error())

View File

@@ -12,10 +12,10 @@ import (
type podPriorityClassForbiddenError struct {
priorityClassName string
spec api.AllowedListSpec
spec api.SelectorAllowedListSpec
}
func NewPodPriorityClassForbidden(priorityClassName string, spec api.AllowedListSpec) error {
func NewPodPriorityClassForbidden(priorityClassName string, spec api.SelectorAllowedListSpec) error {
return &podPriorityClassForbiddenError{
priorityClassName: priorityClassName,
spec: spec,
@@ -35,6 +35,10 @@ func (f podPriorityClassForbiddenError) Error() (err string) {
extra = append(extra, fmt.Sprintf(" use one matching the following regex (%s)", f.spec.Regex))
}
if len(f.spec.Selector.MatchLabels) > 0 || len(f.spec.Selector.MatchExpressions) > 0 {
extra = append(extra, ", or matching the label selector defined in the Tenant")
}
err += strings.Join(extra, " or ")
return

View File

@@ -11,17 +11,17 @@ import (
)
type storageClassNotValidError struct {
spec api.AllowedListSpec
spec api.SelectorAllowedListSpec
}
func NewStorageClassNotValid(storageClasses api.AllowedListSpec) error {
func NewStorageClassNotValid(storageClasses api.SelectorAllowedListSpec) error {
return &storageClassNotValidError{
spec: storageClasses,
}
}
// nolint:predeclared
func appendError(spec api.AllowedListSpec) (append string) {
func appendError(spec api.SelectorAllowedListSpec) (append string) {
if len(spec.Exact) > 0 {
append += fmt.Sprintf(", one of the following (%s)", strings.Join(spec.Exact, ", "))
}
@@ -30,6 +30,10 @@ func appendError(spec api.AllowedListSpec) (append string) {
append += fmt.Sprintf(", or matching the regex %s", spec.Regex)
}
if len(spec.Selector.MatchLabels) > 0 || len(spec.Selector.MatchExpressions) > 0 {
append += ", or matching the label selector defined in the Tenant"
}
return
}
@@ -39,10 +43,10 @@ func (s storageClassNotValidError) Error() (err string) {
type storageClassForbiddenError struct {
className string
spec api.AllowedListSpec
spec api.SelectorAllowedListSpec
}
func NewStorageClassForbidden(className string, storageClasses api.AllowedListSpec) error {
func NewStorageClassForbidden(className string, storageClasses api.SelectorAllowedListSpec) error {
return &storageClassForbiddenError{
className: className,
spec: storageClasses,

View File

@@ -5,9 +5,13 @@ package pvc
import (
"context"
"net/http"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -23,10 +27,22 @@ func Handler() capsulewebhook.Handler {
return &handler{}
}
func (h *handler) getStorageClass(ctx context.Context, c client.Client, name string) (client.Object, error) {
obj := &v1.StorageClass{}
if err := c.Get(ctx, types.NamespacedName{Name: name}, obj); err != nil {
if errors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
return obj, nil
}
func (h *handler) OnCreate(c client.Client, decoder *admission.Decoder, recorder record.EventRecorder) capsulewebhook.Func {
return func(ctx context.Context, req admission.Request) *admission.Response {
var valid, matched bool
pvc := &corev1.PersistentVolumeClaim{}
if err := decoder.Decode(req, pvc); err != nil {
return utils.ErroredResponse(err)
@@ -58,7 +74,25 @@ func (h *handler) OnCreate(c client.Client, decoder *admission.Decoder, recorder
}
sc := *pvc.Spec.StorageClassName
if valid, matched = tnt.Spec.StorageClasses.ExactMatch(sc), tnt.Spec.StorageClasses.RegexMatch(sc); !valid && !matched {
scObj, err := h.getStorageClass(ctx, c, sc)
if err != nil {
response := admission.Errored(http.StatusInternalServerError, err)
return &response
}
var valid, regex, match bool
valid, regex = tnt.Spec.StorageClasses.ExactMatch(sc), tnt.Spec.StorageClasses.RegexMatch(sc)
if scObj != nil {
match = tnt.Spec.StorageClasses.SelectorMatch(scObj)
} else {
match = true
}
if !valid && !regex && !match {
recorder.Eventf(&tnt, corev1.EventTypeWarning, "ForbiddenStorageClass", "PersistentVolumeClaim %s/%s StorageClass %s is forbidden for the current Tenant", req.Namespace, req.Name, sc)
response := admission.Denied(NewStorageClassForbidden(*pvc.Spec.StorageClassName, *tnt.Spec.StorageClasses).Error())