From 27c0a9b328e090adb3eaea4b1b0b4059ee9b9f98 Mon Sep 17 00:00:00 2001 From: ctrought <65360454+ctrought@users.noreply.github.com> Date: Fri, 31 Mar 2023 19:05:59 -0400 Subject: [PATCH] feat: additional labelSelector support * add option to configure labelSelectors on configmaps & secrets * support all labelSelectors * label selector tests * legacy support for ':' delimited selectors and wildcards Signed-off-by: Craig Trought --- README.md | 46 +++++++-- .../chart/reloader/templates/deployment.yaml | 7 +- .../kubernetes/chart/reloader/values.yaml | 1 + internal/pkg/cmd/reloader.go | 81 ++++++++++++++-- internal/pkg/controller/controller.go | 22 +++-- internal/pkg/controller/controller_test.go | 97 ++++++++++++++----- internal/pkg/leadership/leadership_test.go | 2 +- 7 files changed, 206 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 44a86884..9e390b52 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,7 @@ spec: - you may override the secret annotation with the `--secret-annotation` flag - you may want to prevent watching certain namespaces with the `--namespaces-to-ignore` flag - you may want to watch only a set of namespaces with certain labels by using the `--namespace-selector` flag +- you may want to watch only a set of secrets/configmaps with certain labels by using the `--resource-label-selector` flag - you may want to prevent watching certain resources with the `--resources-to-ignore` flag - you can configure logging in JSON format with the `--log-format=json` option - you can configure the "reload strategy" with the `--reload-strategy=` option (details below) @@ -183,9 +184,36 @@ Reloader can be configured to ignore the resources `secrets` and `configmaps` by `Note`: At one time only one of these resource can be ignored, trying to do it will cause error in Reloader. Workaround for ignoring both resources is by scaling down the reloader pods to `0`. -Reloader can be configured to watch only namespaces labeled with (one or more) labels of your choosing by using the `--namespace-selector` parameter, for example: +Reloader can be configured to watch only secrets/configmaps labeled with (one or more) labels of your choosing by using the `--resource-label-selector` parameter. Supported operators are `!, in, notin, ==, =, !=`, if no operator is found the 'exists' operator is inferred (ie. key only). The `:` delimited key value mappings are deprecated and if provided will be translated to key=value. Likewise, if a wildcard is provided (e.g. `key:*`) it will be translated to just the standalone `key` which checks for key existence. + +These can be combined together, for example: + ``` ---namespace-selector=reloder:enabled,test:true +--resource-label-selector=reloader=enabled,key-exists,another-label in (value1, value2, value3) +``` + +Only configmaps or secrets labeled like the following will be watched: +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + ... + labels: + reloader: enabled + key-exists: yes + another-label: value1 + + ... +``` + +If you want to select namespace only by the key of the label use ```*``` as the value. +For example, for ```--namespace-selector=select-this:*``` all namespaces with label-key "select-this" will be selected regardless of the labels value + +Reloader can be configured to watch only namespaces labeled with (one or more) labels of your choosing by using the `--namespace-selector` parameter. Reloader can be configured to watch only secrets/configmaps labeled with (one or more) labels of your choosing by using the `--resource-label-selector` parameter. Supported operators are `!, in, notin, ==, =, !=`, if no operator is found the 'exists' operator is inferred (ie. key only). The `:` delimited key value mappings are deprecated and if provided will be translated to key=value. Likewise, if a wildcard is provided (e.g. `key:*`) it will be translated to just the standalone `key` which checks for key existence. + +These can be combined together, for example: +``` +--namespace-selector=reloader:enabled,test:true ``` Only namespaces labeled like the following namespace YAML will be watched: @@ -195,7 +223,7 @@ apiVersion: v1 metadata: ... labels: - reloder: enabled + reloader: enabled test: true ... ``` @@ -255,9 +283,15 @@ Reloader can be configured to ignore the resources `secrets` and `configmaps` by Reloader can be configured to watch only namespaces labeled with (one or more) labels of your choosing by using the `namespaceSelector` parameter -| Parameter | Description | Type | -| ---------------- | -------------------------------------------------------------- | ------- | -| namespaceSelector | list of comma separated key:value namespace | string | +| Parameter | Description | Type | +| ---------------- | ---------------------------------------------------------------------------------- | ------- | +| namespaceSelector | list of comma separated label selectors, if mulitple are provided they are ANDed | string | + +Reloader can be configured to watch only configmaps/secrets labeled with (one or more) labels of your choosing by using the `resourceLabelSelector` parameter + +| Parameter | Description | Type | +| ---------------------- | ---------------------------------------------------------------------------------- | ------- | +| resourceLabelSelector | list of comma separated label selectors, if mulitple are provided they are ANDed | string | You can also set the log format of Reloader to json by setting `logFormat` to `json` in values.yaml and apply the chart diff --git a/deployments/kubernetes/chart/reloader/templates/deployment.yaml b/deployments/kubernetes/chart/reloader/templates/deployment.yaml index c7a093c9..32ab6d6f 100644 --- a/deployments/kubernetes/chart/reloader/templates/deployment.yaml +++ b/deployments/kubernetes/chart/reloader/templates/deployment.yaml @@ -159,7 +159,7 @@ spec: - mountPath: /tmp/ name: tmp-volume {{- end }} - {{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA)}} + {{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA)}} args: {{- if .Values.reloader.logFormat }} - "--log-format={{ .Values.reloader.logFormat }}" @@ -175,7 +175,10 @@ spec: {{- end }} {{- if .Values.reloader.namespaceSelector }} - "--namespace-selector={{ .Values.reloader.namespaceSelector }}" - {{- end }} + {{- end }} + {{- if .Values.reloader.resourceLabelSelector }} + - "--resource-label-selector={{ .Values.reloader.resourceLabelSelector }}" + {{- end }} {{- if .Values.reloader.custom_annotations }} {{- if .Values.reloader.custom_annotations.configmap }} - "--configmap-annotation" diff --git a/deployments/kubernetes/chart/reloader/values.yaml b/deployments/kubernetes/chart/reloader/values.yaml index 1cc4fa35..5f9ab209 100644 --- a/deployments/kubernetes/chart/reloader/values.yaml +++ b/deployments/kubernetes/chart/reloader/values.yaml @@ -21,6 +21,7 @@ reloader: reloadStrategy: default # Set to default, env-vars or annotations ignoreNamespaces: "" # Comma separated list of namespaces to ignore namespaceSelector: "" # Comma separated list of 'key:value' labels for namespaces selection + resourceLabelSelector: "" # Comma separated list of 'key:value' labels for configmap/secret selection logFormat: "" #json watchGlobally: true # Set to true to enable leadership election allowing you to run multiple replicas diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index a0ed7ea3..0158be14 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -39,7 +39,8 @@ func NewReloaderCommand() *cobra.Command { cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON") cmd.PersistentFlags().StringSlice("resources-to-ignore", []string{}, "list of resources to ignore (valid options 'configMaps' or 'secrets')") cmd.PersistentFlags().StringSlice("namespaces-to-ignore", []string{}, "list of namespaces to ignore") - cmd.PersistentFlags().StringSlice("namespace-selector", []string{}, "list of key:vaule namespace labels to include") + cmd.PersistentFlags().StringSlice("namespace-selector", []string{}, "list of key:value labels to filter on for namespaces") + cmd.PersistentFlags().StringSlice("resource-label-selector", []string{}, "list of key:value labels to filter on for configmaps and secrets") cmd.PersistentFlags().StringVar(&options.IsArgoRollouts, "is-Argo-Rollouts", "false", "Add support for argo rollouts") cmd.PersistentFlags().StringVar(&options.ReloadStrategy, constants.ReloadStrategyFlag, constants.EnvVarsReloadStrategy, "Specifies the desired reload strategy") cmd.PersistentFlags().StringVar(&options.ReloadOnCreate, "reload-on-create", "false", "Add support to watch create events") @@ -140,8 +141,17 @@ func startReloader(cmd *cobra.Command, args []string) { logrus.Fatal(err) } + resourceLabelSelector, err := getResourceLabelSelector(cmd) + if err != nil { + logrus.Fatal(err) + } + if len(namespaceLabelSelector) > 0 { - logrus.Warnf("namespace-selector is set, will detect changes in namespaces with these labels: %s.", namespaceLabelSelector) + logrus.Warnf("namespace-selector is set, will only detect changes in namespaces with these labels: %s.", namespaceLabelSelector) + } + + if len(resourceLabelSelector) > 0 { + logrus.Warnf("resource-label-selector is set, will only detect changes on resources with these labels: %s.", resourceLabelSelector) } collectors := metrics.SetupPrometheusEndpoint() @@ -152,7 +162,7 @@ func startReloader(cmd *cobra.Command, args []string) { continue } - c, err := controller.NewController(clientset, k, currentNamespace, ignoredNamespacesList, namespaceLabelSelector, collectors) + c, err := controller.NewController(clientset, k, currentNamespace, ignoredNamespacesList, namespaceLabelSelector, resourceLabelSelector, collectors) if err != nil { logrus.Fatalf("%s", err) } @@ -187,19 +197,72 @@ func getIgnoredNamespacesList(cmd *cobra.Command) (util.List, error) { return getStringSliceFromFlags(cmd, "namespaces-to-ignore") } -func getNamespaceLabelSelector(cmd *cobra.Command) (util.Map, error) { +func getNamespaceLabelSelector(cmd *cobra.Command) (string, error) { slice, err := getStringSliceFromFlags(cmd, "namespace-selector") if err != nil { logrus.Fatal(err) } - var namespaceSelectorMap util.Map = make(util.Map) - for _, kv := range slice { - split := strings.Split(kv, ":") - namespaceSelectorMap[split[0]] = split[1] + for i, kv := range slice { + // Legacy support for ":" as a delimiter and "*" for wildcard. + if strings.Contains(kv, ":") { + split := strings.Split(kv, ":") + if split[1] == "*" { + slice[i] = split[0] + } else { + slice[i] = split[0] + "=" + split[1] + } + } + // Convert wildcard to valid apimachinery operator + if strings.Contains(kv, "=") { + split := strings.Split(kv, "=") + if split[1] == "*" { + slice[i] = split[0] + } + } } - return namespaceSelectorMap, nil + namespaceLabelSelector := strings.Join(slice[:], ",") + _, err = v1.ParseToLabelSelector(namespaceLabelSelector) + if err != nil { + logrus.Fatal(err) + } + + return namespaceLabelSelector, nil +} + +func getResourceLabelSelector(cmd *cobra.Command) (string, error) { + slice, err := getStringSliceFromFlags(cmd, "resource-label-selector") + if err != nil { + logrus.Fatal(err) + } + + for i, kv := range slice { + // Legacy support for ":" as a delimiter and "*" for wildcard. + if strings.Contains(kv, ":") { + split := strings.Split(kv, ":") + if split[1] == "*" { + slice[i] = split[0] + } else { + slice[i] = split[0] + "=" + split[1] + } + } + // Convert wildcard to valid apimachinery operator + if strings.Contains(kv, "=") { + split := strings.Split(kv, "=") + if split[1] == "*" { + slice[i] = split[0] + } + } + } + + resourceLabelSelector := strings.Join(slice[:], ",") + _, err = v1.ParseToLabelSelector(resourceLabelSelector) + if err != nil { + logrus.Fatal(err) + } + + return resourceLabelSelector, nil } func getStringSliceFromFlags(cmd *cobra.Command, flag string) ([]string, error) { diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index a0e5a666..1b380ecd 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -13,7 +13,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -36,7 +35,8 @@ type Controller struct { ignoredNamespaces util.List collectors metrics.Collectors recorder record.EventRecorder - namespaceSelector map[string]string + namespaceSelector string + resourceSelector string } // controllerInitialized flag determines whether controlled is being initialized @@ -46,7 +46,7 @@ var selectedNamespacesCache []string // NewController for initializing a Controller func NewController( - client kubernetes.Interface, resource string, namespace string, ignoredNamespaces []string, namespaceLabelSelector map[string]string, collectors metrics.Collectors) (*Controller, error) { + client kubernetes.Interface, resource string, namespace string, ignoredNamespaces []string, namespaceLabelSelector string, resourceLabelSelector string, collectors metrics.Collectors) (*Controller, error) { if options.SyncAfterRestart { secretControllerInitialized = true @@ -58,6 +58,7 @@ func NewController( namespace: namespace, ignoredNamespaces: ignoredNamespaces, namespaceSelector: namespaceLabelSelector, + resourceSelector: resourceLabelSelector, resource: resource, } eventBroadcaster := record.NewBroadcaster() @@ -70,8 +71,9 @@ func NewController( optionsModifier := func(options *metav1.ListOptions) { if resource == "namespaces" { - labelSelector := metav1.LabelSelector{MatchLabels: c.namespaceSelector} - options.LabelSelector = labels.Set(labelSelector.MatchLabels).String() + options.LabelSelector = c.namespaceSelector + } else if len(c.resourceSelector) > 0 { + options.LabelSelector = c.resourceSelector } else { options.FieldSelector = fields.Everything().String() } @@ -99,7 +101,7 @@ func (c *Controller) Add(obj interface{}) { switch object := obj.(type) { case *v1.Namespace: - c.addSelectedNamespaceToCache(object) + c.addSelectedNamespaceToCache(*object) return } @@ -142,14 +144,16 @@ func (c *Controller) resourceInSelectedNamespaces(raw interface{}) bool { return false } -func (c *Controller) addSelectedNamespaceToCache(namespace *v1.Namespace) { +func (c *Controller) addSelectedNamespaceToCache(namespace v1.Namespace) { selectedNamespacesCache = append(selectedNamespacesCache, namespace.GetName()) + logrus.Infof("added namespace to be watched: %s", namespace.GetName()) } -func (c *Controller) removeSelectedNamespaceFromCache(namespace *v1.Namespace) { +func (c *Controller) removeSelectedNamespaceFromCache(namespace v1.Namespace) { for i, v := range selectedNamespacesCache { if v == namespace.GetName() { selectedNamespacesCache = append(selectedNamespacesCache[:i], selectedNamespacesCache[i+1:]...) + logrus.Infof("removed namespace from watch: %s", namespace.GetName()) return } } @@ -176,7 +180,7 @@ func (c *Controller) Update(old interface{}, new interface{}) { func (c *Controller) Delete(old interface{}) { switch object := old.(type) { case *v1.Namespace: - c.removeSelectedNamespaceFromCache(object) + c.removeSelectedNamespaceFromCache(*object) return } diff --git a/internal/pkg/controller/controller_test.go b/internal/pkg/controller/controller_test.go index 6b9179bc..62a9370e 100644 --- a/internal/pkg/controller/controller_test.go +++ b/internal/pkg/controller/controller_test.go @@ -45,7 +45,7 @@ func TestMain(m *testing.M) { logrus.Infof("Creating controller") for k := range kube.ResourceMap { - c, err := NewController(clients.KubernetesClient, k, namespace, []string{}, map[string]string{}, collectors) + c, err := NewController(clients.KubernetesClient, k, namespace, []string{}, "", "", collectors) if err != nil { logrus.Fatalf("%s", err) } @@ -2291,7 +2291,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { queue workqueue.RateLimitingInterface informer cache.Controller namespace v1.Namespace - namespaceSelector util.Map + namespaceSelector string } type args struct { raw interface{} @@ -2305,10 +2305,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { { name: "TestConfigMapResourceInNamespaceSelector", fields: fields{ - namespaceSelector: util.Map{ - "select": "this", - "select2": "this2", - }, + namespaceSelector: "select=this,select2=this2", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "selected-namespace", @@ -2326,10 +2323,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { }, { name: "TestConfigMapResourceNotInNamespaceSelector", fields: fields{ - namespaceSelector: util.Map{ - "select": "this", - "select2": "this2", - }, + namespaceSelector: "select=this,select2=this2", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "not-selected-namespace", @@ -2345,10 +2339,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { { name: "TestSecretResourceInNamespaceSelector", fields: fields{ - namespaceSelector: util.Map{ - "select": "this", - "select2": "this2", - }, + namespaceSelector: "select=this,select2=this2", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "selected-namespace", @@ -2366,10 +2357,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { }, { name: "TestSecretResourceNotInNamespaceSelector", fields: fields{ - namespaceSelector: util.Map{ - "select": "this", - "select2": "this2", - }, + namespaceSelector: "select=this,select2=this2", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "not-selected-namespace", @@ -2382,11 +2370,9 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { }, want: false, }, { - name: "TestSecretResourceInNamespaceSelectorWiledcardValue", + name: "TestSecretResourceInNamespaceSelectorKeyExists", fields: fields{ - namespaceSelector: util.Map{ - "select": "*", - }, + namespaceSelector: "select", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "selected-namespace", @@ -2400,6 +2386,59 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { raw: testutil.GetSecret("selected-namespace", "secret", "test"), }, want: true, + }, { + name: "TestSecretResourceInNamespaceSelectorValueIn", + fields: fields{ + namespaceSelector: "select in (select1, select2, select3)", + namespace: v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "selected-namespace", + Labels: map[string]string{ + "select": "select2", + }, + }, + }, + }, + args: args{ + raw: testutil.GetSecret("selected-namespace", "secret", "test"), + }, + want: true, + }, { + name: "TestSecretResourceInNamespaceSelectorKeyDoesNotExist", + fields: fields{ + namespaceSelector: "!select2", + namespace: v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "selected-namespace", + Labels: map[string]string{ + "select": "this", + }, + }, + }, + }, + args: args{ + raw: testutil.GetSecret("selected-namespace", "secret", "test"), + }, + want: true, + }, { + name: "TestSecretResourceInNamespaceSelectorMultipleConditions", + fields: fields{ + namespaceSelector: "select,select2=this2,select3!=this4", + namespace: v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "selected-namespace", + Labels: map[string]string{ + "select": "this", + "select2": "this2", + "select3": "this3", + }, + }, + }, + }, + args: args{ + raw: testutil.GetSecret("selected-namespace", "secret", "test"), + }, + want: true, }, } @@ -2418,9 +2457,21 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { namespaceSelector: tt.fields.namespaceSelector, } - if got := c.resourceInNamespaceSelector(tt.args.raw); got != tt.want { + listOptions := metav1.ListOptions{} + listOptions.LabelSelector = tt.fields.namespaceSelector + namespaces, _ := fakeClient.CoreV1().Namespaces().List(context.Background(), listOptions) + + for _, ns := range namespaces.Items { + c.addSelectedNamespaceToCache(ns) + } + + if got := c.resourceInSelectedNamespaces(tt.args.raw); got != tt.want { t.Errorf("Controller.resourceInNamespaceSelector() = %v, want %v", got, tt.want) } + + for _, ns := range namespaces.Items { + c.removeSelectedNamespaceFromCache(ns) + } }) } } diff --git a/internal/pkg/leadership/leadership_test.go b/internal/pkg/leadership/leadership_test.go index 99f638c2..8563803e 100644 --- a/internal/pkg/leadership/leadership_test.go +++ b/internal/pkg/leadership/leadership_test.go @@ -119,7 +119,7 @@ func TestRunLeaderElectionWithControllers(t *testing.T) { t.Logf("Creating controller") var controllers []*controller.Controller for k := range kube.ResourceMap { - c, err := controller.NewController(testutil.Clients.KubernetesClient, k, testutil.Namespace, []string{}, map[string]string{}, metrics.NewCollectors()) + c, err := controller.NewController(testutil.Clients.KubernetesClient, k, testutil.Namespace, []string{}, map[string]string{}, map[string]string{}, metrics.NewCollectors()) if err != nil { logrus.Fatalf("%s", err) }