From e39a8f6bcf6c6288450a507fa59ced088d99a2f6 Mon Sep 17 00:00:00 2001 From: ctrought <65360454+ctrought@users.noreply.github.com> Date: Fri, 31 Mar 2023 14:43:33 -0400 Subject: [PATCH 1/8] refactor: use Namespace controller for namespaceSelector Signed-off-by: Craig Trought --- internal/pkg/cmd/reloader.go | 2 +- internal/pkg/controller/controller.go | 75 ++++++++++++++++++--------- pkg/kube/resourcemapper.go | 1 + 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index b297ecad..a0ed7ea3 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -148,7 +148,7 @@ func startReloader(cmd *cobra.Command, args []string) { var controllers []*controller.Controller for k := range kube.ResourceMap { - if ignoredResourcesList.Contains(k) { + if ignoredResourcesList.Contains(k) || (len(namespaceLabelSelector) == 0 && k == "namespaces") { continue } diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index f1934ffc..a0e5a666 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -1,7 +1,6 @@ package controller import ( - "context" "fmt" "time" @@ -14,6 +13,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/kubectl/pkg/scheme" + "k8s.io/utils/strings/slices" ) // Controller for checking events @@ -41,6 +42,7 @@ type Controller struct { // controllerInitialized flag determines whether controlled is being initialized var secretControllerInitialized bool = false var configmapControllerInitialized bool = false +var selectedNamespacesCache []string // NewController for initializing a Controller func NewController( @@ -65,7 +67,17 @@ func NewController( recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("reloader-%s", resource)}) queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) - listWatcher := cache.NewListWatchFromClient(client.CoreV1().RESTClient(), resource, namespace, fields.Everything()) + + optionsModifier := func(options *metav1.ListOptions) { + if resource == "namespaces" { + labelSelector := metav1.LabelSelector{MatchLabels: c.namespaceSelector} + options.LabelSelector = labels.Set(labelSelector.MatchLabels).String() + } else { + options.FieldSelector = fields.Everything().String() + } + } + + listWatcher := cache.NewFilteredListWatchFromClient(client.CoreV1().RESTClient(), resource, namespace, optionsModifier) indexer, informer := cache.NewIndexerInformer(listWatcher, kube.ResourceMap[resource], 0, cache.ResourceEventHandlerFuncs{ AddFunc: c.Add, @@ -84,8 +96,15 @@ func NewController( // Add function to add a new object to the queue in case of creating a resource func (c *Controller) Add(obj interface{}) { + + switch object := obj.(type) { + case *v1.Namespace: + c.addSelectedNamespaceToCache(object) + return + } + if options.ReloadOnCreate == "true" { - if !c.resourceInIgnoredNamespace(obj) && c.resourceInNamespaceSelector(obj) && secretControllerInitialized && configmapControllerInitialized { + if !c.resourceInIgnoredNamespace(obj) && c.resourceInSelectedNamespaces(obj) && secretControllerInitialized && configmapControllerInitialized { c.queue.Add(handler.ResourceCreatedHandler{ Resource: obj, Collectors: c.collectors, @@ -105,45 +124,45 @@ func (c *Controller) resourceInIgnoredNamespace(raw interface{}) bool { return false } -func (c *Controller) resourceInNamespaceSelector(raw interface{}) bool { +func (c *Controller) resourceInSelectedNamespaces(raw interface{}) bool { if len(c.namespaceSelector) == 0 { return true } switch object := raw.(type) { case *v1.ConfigMap: - return c.matchLabels(object.ObjectMeta.Namespace) + if slices.Contains(selectedNamespacesCache, object.GetNamespace()) { + return true + } case *v1.Secret: - return c.matchLabels(object.ObjectMeta.Namespace) + if slices.Contains(selectedNamespacesCache, object.GetNamespace()) { + return true + } } - return true + return false } -func (c *Controller) matchLabels(resourceNamespace string) bool { - namespace, err := c.client.CoreV1().Namespaces().Get(context.Background(), resourceNamespace, metav1.GetOptions{}) - if err != nil { - logrus.Warn(err) - return false - } +func (c *Controller) addSelectedNamespaceToCache(namespace *v1.Namespace) { + selectedNamespacesCache = append(selectedNamespacesCache, namespace.GetName()) +} - for selectorKey, selectorVal := range c.namespaceSelector { - - namespaceLabelVal, namespaceLabelKeyExists := namespace.ObjectMeta.Labels[selectorKey] - - if namespaceLabelKeyExists && selectorVal == "*" { - continue - } - - if !namespaceLabelKeyExists || selectorVal != namespaceLabelVal { - return false +func (c *Controller) removeSelectedNamespaceFromCache(namespace *v1.Namespace) { + for i, v := range selectedNamespacesCache { + if v == namespace.GetName() { + selectedNamespacesCache = append(selectedNamespacesCache[:i], selectedNamespacesCache[i+1:]...) + return } } - return true } // Update function to add an old object and a new object to the queue in case of updating a resource func (c *Controller) Update(old interface{}, new interface{}) { - if !c.resourceInIgnoredNamespace(new) && c.resourceInNamespaceSelector(new) { + switch new.(type) { + case *v1.Namespace: + return + } + + if !c.resourceInIgnoredNamespace(new) && c.resourceInSelectedNamespaces(new) { c.queue.Add(handler.ResourceUpdatedHandler{ Resource: new, OldResource: old, @@ -155,6 +174,12 @@ func (c *Controller) Update(old interface{}, new interface{}) { // Delete function to add an object to the queue in case of deleting a resource func (c *Controller) Delete(old interface{}) { + switch object := old.(type) { + case *v1.Namespace: + c.removeSelectedNamespaceFromCache(object) + return + } + // Todo: Any future delete event can be handled here } diff --git a/pkg/kube/resourcemapper.go b/pkg/kube/resourcemapper.go index 2d82ca28..fb42e61f 100644 --- a/pkg/kube/resourcemapper.go +++ b/pkg/kube/resourcemapper.go @@ -9,4 +9,5 @@ import ( var ResourceMap = map[string]runtime.Object{ "configMaps": &v1.ConfigMap{}, "secrets": &v1.Secret{}, + "namespaces": &v1.Namespace{}, } From 27c0a9b328e090adb3eaea4b1b0b4059ee9b9f98 Mon Sep 17 00:00:00 2001 From: ctrought <65360454+ctrought@users.noreply.github.com> Date: Fri, 31 Mar 2023 19:05:59 -0400 Subject: [PATCH 2/8] feat: additional labelSelector support * add option to configure labelSelectors on configmaps & secrets * support all labelSelectors * label selector tests * legacy support for ':' delimited selectors and wildcards Signed-off-by: Craig Trought --- README.md | 46 +++++++-- .../chart/reloader/templates/deployment.yaml | 7 +- .../kubernetes/chart/reloader/values.yaml | 1 + internal/pkg/cmd/reloader.go | 81 ++++++++++++++-- internal/pkg/controller/controller.go | 22 +++-- internal/pkg/controller/controller_test.go | 97 ++++++++++++++----- internal/pkg/leadership/leadership_test.go | 2 +- 7 files changed, 206 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 44a86884..9e390b52 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,7 @@ spec: - you may override the secret annotation with the `--secret-annotation` flag - you may want to prevent watching certain namespaces with the `--namespaces-to-ignore` flag - you may want to watch only a set of namespaces with certain labels by using the `--namespace-selector` flag +- you may want to watch only a set of secrets/configmaps with certain labels by using the `--resource-label-selector` flag - you may want to prevent watching certain resources with the `--resources-to-ignore` flag - you can configure logging in JSON format with the `--log-format=json` option - you can configure the "reload strategy" with the `--reload-strategy=` option (details below) @@ -183,9 +184,36 @@ Reloader can be configured to ignore the resources `secrets` and `configmaps` by `Note`: At one time only one of these resource can be ignored, trying to do it will cause error in Reloader. Workaround for ignoring both resources is by scaling down the reloader pods to `0`. -Reloader can be configured to watch only namespaces labeled with (one or more) labels of your choosing by using the `--namespace-selector` parameter, for example: +Reloader can be configured to watch only secrets/configmaps labeled with (one or more) labels of your choosing by using the `--resource-label-selector` parameter. Supported operators are `!, in, notin, ==, =, !=`, if no operator is found the 'exists' operator is inferred (ie. key only). The `:` delimited key value mappings are deprecated and if provided will be translated to key=value. Likewise, if a wildcard is provided (e.g. `key:*`) it will be translated to just the standalone `key` which checks for key existence. + +These can be combined together, for example: + ``` ---namespace-selector=reloder:enabled,test:true +--resource-label-selector=reloader=enabled,key-exists,another-label in (value1, value2, value3) +``` + +Only configmaps or secrets labeled like the following will be watched: +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + ... + labels: + reloader: enabled + key-exists: yes + another-label: value1 + + ... +``` + +If you want to select namespace only by the key of the label use ```*``` as the value. +For example, for ```--namespace-selector=select-this:*``` all namespaces with label-key "select-this" will be selected regardless of the labels value + +Reloader can be configured to watch only namespaces labeled with (one or more) labels of your choosing by using the `--namespace-selector` parameter. Reloader can be configured to watch only secrets/configmaps labeled with (one or more) labels of your choosing by using the `--resource-label-selector` parameter. Supported operators are `!, in, notin, ==, =, !=`, if no operator is found the 'exists' operator is inferred (ie. key only). The `:` delimited key value mappings are deprecated and if provided will be translated to key=value. Likewise, if a wildcard is provided (e.g. `key:*`) it will be translated to just the standalone `key` which checks for key existence. + +These can be combined together, for example: +``` +--namespace-selector=reloader:enabled,test:true ``` Only namespaces labeled like the following namespace YAML will be watched: @@ -195,7 +223,7 @@ apiVersion: v1 metadata: ... labels: - reloder: enabled + reloader: enabled test: true ... ``` @@ -255,9 +283,15 @@ Reloader can be configured to ignore the resources `secrets` and `configmaps` by Reloader can be configured to watch only namespaces labeled with (one or more) labels of your choosing by using the `namespaceSelector` parameter -| Parameter | Description | Type | -| ---------------- | -------------------------------------------------------------- | ------- | -| namespaceSelector | list of comma separated key:value namespace | string | +| Parameter | Description | Type | +| ---------------- | ---------------------------------------------------------------------------------- | ------- | +| namespaceSelector | list of comma separated label selectors, if mulitple are provided they are ANDed | string | + +Reloader can be configured to watch only configmaps/secrets labeled with (one or more) labels of your choosing by using the `resourceLabelSelector` parameter + +| Parameter | Description | Type | +| ---------------------- | ---------------------------------------------------------------------------------- | ------- | +| resourceLabelSelector | list of comma separated label selectors, if mulitple are provided they are ANDed | string | You can also set the log format of Reloader to json by setting `logFormat` to `json` in values.yaml and apply the chart diff --git a/deployments/kubernetes/chart/reloader/templates/deployment.yaml b/deployments/kubernetes/chart/reloader/templates/deployment.yaml index c7a093c9..32ab6d6f 100644 --- a/deployments/kubernetes/chart/reloader/templates/deployment.yaml +++ b/deployments/kubernetes/chart/reloader/templates/deployment.yaml @@ -159,7 +159,7 @@ spec: - mountPath: /tmp/ name: tmp-volume {{- end }} - {{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA)}} + {{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA)}} args: {{- if .Values.reloader.logFormat }} - "--log-format={{ .Values.reloader.logFormat }}" @@ -175,7 +175,10 @@ spec: {{- end }} {{- if .Values.reloader.namespaceSelector }} - "--namespace-selector={{ .Values.reloader.namespaceSelector }}" - {{- end }} + {{- end }} + {{- if .Values.reloader.resourceLabelSelector }} + - "--resource-label-selector={{ .Values.reloader.resourceLabelSelector }}" + {{- end }} {{- if .Values.reloader.custom_annotations }} {{- if .Values.reloader.custom_annotations.configmap }} - "--configmap-annotation" diff --git a/deployments/kubernetes/chart/reloader/values.yaml b/deployments/kubernetes/chart/reloader/values.yaml index 1cc4fa35..5f9ab209 100644 --- a/deployments/kubernetes/chart/reloader/values.yaml +++ b/deployments/kubernetes/chart/reloader/values.yaml @@ -21,6 +21,7 @@ reloader: reloadStrategy: default # Set to default, env-vars or annotations ignoreNamespaces: "" # Comma separated list of namespaces to ignore namespaceSelector: "" # Comma separated list of 'key:value' labels for namespaces selection + resourceLabelSelector: "" # Comma separated list of 'key:value' labels for configmap/secret selection logFormat: "" #json watchGlobally: true # Set to true to enable leadership election allowing you to run multiple replicas diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index a0ed7ea3..0158be14 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -39,7 +39,8 @@ func NewReloaderCommand() *cobra.Command { cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON") cmd.PersistentFlags().StringSlice("resources-to-ignore", []string{}, "list of resources to ignore (valid options 'configMaps' or 'secrets')") cmd.PersistentFlags().StringSlice("namespaces-to-ignore", []string{}, "list of namespaces to ignore") - cmd.PersistentFlags().StringSlice("namespace-selector", []string{}, "list of key:vaule namespace labels to include") + cmd.PersistentFlags().StringSlice("namespace-selector", []string{}, "list of key:value labels to filter on for namespaces") + cmd.PersistentFlags().StringSlice("resource-label-selector", []string{}, "list of key:value labels to filter on for configmaps and secrets") cmd.PersistentFlags().StringVar(&options.IsArgoRollouts, "is-Argo-Rollouts", "false", "Add support for argo rollouts") cmd.PersistentFlags().StringVar(&options.ReloadStrategy, constants.ReloadStrategyFlag, constants.EnvVarsReloadStrategy, "Specifies the desired reload strategy") cmd.PersistentFlags().StringVar(&options.ReloadOnCreate, "reload-on-create", "false", "Add support to watch create events") @@ -140,8 +141,17 @@ func startReloader(cmd *cobra.Command, args []string) { logrus.Fatal(err) } + resourceLabelSelector, err := getResourceLabelSelector(cmd) + if err != nil { + logrus.Fatal(err) + } + if len(namespaceLabelSelector) > 0 { - logrus.Warnf("namespace-selector is set, will detect changes in namespaces with these labels: %s.", namespaceLabelSelector) + logrus.Warnf("namespace-selector is set, will only detect changes in namespaces with these labels: %s.", namespaceLabelSelector) + } + + if len(resourceLabelSelector) > 0 { + logrus.Warnf("resource-label-selector is set, will only detect changes on resources with these labels: %s.", resourceLabelSelector) } collectors := metrics.SetupPrometheusEndpoint() @@ -152,7 +162,7 @@ func startReloader(cmd *cobra.Command, args []string) { continue } - c, err := controller.NewController(clientset, k, currentNamespace, ignoredNamespacesList, namespaceLabelSelector, collectors) + c, err := controller.NewController(clientset, k, currentNamespace, ignoredNamespacesList, namespaceLabelSelector, resourceLabelSelector, collectors) if err != nil { logrus.Fatalf("%s", err) } @@ -187,19 +197,72 @@ func getIgnoredNamespacesList(cmd *cobra.Command) (util.List, error) { return getStringSliceFromFlags(cmd, "namespaces-to-ignore") } -func getNamespaceLabelSelector(cmd *cobra.Command) (util.Map, error) { +func getNamespaceLabelSelector(cmd *cobra.Command) (string, error) { slice, err := getStringSliceFromFlags(cmd, "namespace-selector") if err != nil { logrus.Fatal(err) } - var namespaceSelectorMap util.Map = make(util.Map) - for _, kv := range slice { - split := strings.Split(kv, ":") - namespaceSelectorMap[split[0]] = split[1] + for i, kv := range slice { + // Legacy support for ":" as a delimiter and "*" for wildcard. + if strings.Contains(kv, ":") { + split := strings.Split(kv, ":") + if split[1] == "*" { + slice[i] = split[0] + } else { + slice[i] = split[0] + "=" + split[1] + } + } + // Convert wildcard to valid apimachinery operator + if strings.Contains(kv, "=") { + split := strings.Split(kv, "=") + if split[1] == "*" { + slice[i] = split[0] + } + } } - return namespaceSelectorMap, nil + namespaceLabelSelector := strings.Join(slice[:], ",") + _, err = v1.ParseToLabelSelector(namespaceLabelSelector) + if err != nil { + logrus.Fatal(err) + } + + return namespaceLabelSelector, nil +} + +func getResourceLabelSelector(cmd *cobra.Command) (string, error) { + slice, err := getStringSliceFromFlags(cmd, "resource-label-selector") + if err != nil { + logrus.Fatal(err) + } + + for i, kv := range slice { + // Legacy support for ":" as a delimiter and "*" for wildcard. + if strings.Contains(kv, ":") { + split := strings.Split(kv, ":") + if split[1] == "*" { + slice[i] = split[0] + } else { + slice[i] = split[0] + "=" + split[1] + } + } + // Convert wildcard to valid apimachinery operator + if strings.Contains(kv, "=") { + split := strings.Split(kv, "=") + if split[1] == "*" { + slice[i] = split[0] + } + } + } + + resourceLabelSelector := strings.Join(slice[:], ",") + _, err = v1.ParseToLabelSelector(resourceLabelSelector) + if err != nil { + logrus.Fatal(err) + } + + return resourceLabelSelector, nil } func getStringSliceFromFlags(cmd *cobra.Command, flag string) ([]string, error) { diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index a0e5a666..1b380ecd 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -13,7 +13,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -36,7 +35,8 @@ type Controller struct { ignoredNamespaces util.List collectors metrics.Collectors recorder record.EventRecorder - namespaceSelector map[string]string + namespaceSelector string + resourceSelector string } // controllerInitialized flag determines whether controlled is being initialized @@ -46,7 +46,7 @@ var selectedNamespacesCache []string // NewController for initializing a Controller func NewController( - client kubernetes.Interface, resource string, namespace string, ignoredNamespaces []string, namespaceLabelSelector map[string]string, collectors metrics.Collectors) (*Controller, error) { + client kubernetes.Interface, resource string, namespace string, ignoredNamespaces []string, namespaceLabelSelector string, resourceLabelSelector string, collectors metrics.Collectors) (*Controller, error) { if options.SyncAfterRestart { secretControllerInitialized = true @@ -58,6 +58,7 @@ func NewController( namespace: namespace, ignoredNamespaces: ignoredNamespaces, namespaceSelector: namespaceLabelSelector, + resourceSelector: resourceLabelSelector, resource: resource, } eventBroadcaster := record.NewBroadcaster() @@ -70,8 +71,9 @@ func NewController( optionsModifier := func(options *metav1.ListOptions) { if resource == "namespaces" { - labelSelector := metav1.LabelSelector{MatchLabels: c.namespaceSelector} - options.LabelSelector = labels.Set(labelSelector.MatchLabels).String() + options.LabelSelector = c.namespaceSelector + } else if len(c.resourceSelector) > 0 { + options.LabelSelector = c.resourceSelector } else { options.FieldSelector = fields.Everything().String() } @@ -99,7 +101,7 @@ func (c *Controller) Add(obj interface{}) { switch object := obj.(type) { case *v1.Namespace: - c.addSelectedNamespaceToCache(object) + c.addSelectedNamespaceToCache(*object) return } @@ -142,14 +144,16 @@ func (c *Controller) resourceInSelectedNamespaces(raw interface{}) bool { return false } -func (c *Controller) addSelectedNamespaceToCache(namespace *v1.Namespace) { +func (c *Controller) addSelectedNamespaceToCache(namespace v1.Namespace) { selectedNamespacesCache = append(selectedNamespacesCache, namespace.GetName()) + logrus.Infof("added namespace to be watched: %s", namespace.GetName()) } -func (c *Controller) removeSelectedNamespaceFromCache(namespace *v1.Namespace) { +func (c *Controller) removeSelectedNamespaceFromCache(namespace v1.Namespace) { for i, v := range selectedNamespacesCache { if v == namespace.GetName() { selectedNamespacesCache = append(selectedNamespacesCache[:i], selectedNamespacesCache[i+1:]...) + logrus.Infof("removed namespace from watch: %s", namespace.GetName()) return } } @@ -176,7 +180,7 @@ func (c *Controller) Update(old interface{}, new interface{}) { func (c *Controller) Delete(old interface{}) { switch object := old.(type) { case *v1.Namespace: - c.removeSelectedNamespaceFromCache(object) + c.removeSelectedNamespaceFromCache(*object) return } diff --git a/internal/pkg/controller/controller_test.go b/internal/pkg/controller/controller_test.go index 6b9179bc..62a9370e 100644 --- a/internal/pkg/controller/controller_test.go +++ b/internal/pkg/controller/controller_test.go @@ -45,7 +45,7 @@ func TestMain(m *testing.M) { logrus.Infof("Creating controller") for k := range kube.ResourceMap { - c, err := NewController(clients.KubernetesClient, k, namespace, []string{}, map[string]string{}, collectors) + c, err := NewController(clients.KubernetesClient, k, namespace, []string{}, "", "", collectors) if err != nil { logrus.Fatalf("%s", err) } @@ -2291,7 +2291,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { queue workqueue.RateLimitingInterface informer cache.Controller namespace v1.Namespace - namespaceSelector util.Map + namespaceSelector string } type args struct { raw interface{} @@ -2305,10 +2305,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { { name: "TestConfigMapResourceInNamespaceSelector", fields: fields{ - namespaceSelector: util.Map{ - "select": "this", - "select2": "this2", - }, + namespaceSelector: "select=this,select2=this2", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "selected-namespace", @@ -2326,10 +2323,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { }, { name: "TestConfigMapResourceNotInNamespaceSelector", fields: fields{ - namespaceSelector: util.Map{ - "select": "this", - "select2": "this2", - }, + namespaceSelector: "select=this,select2=this2", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "not-selected-namespace", @@ -2345,10 +2339,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { { name: "TestSecretResourceInNamespaceSelector", fields: fields{ - namespaceSelector: util.Map{ - "select": "this", - "select2": "this2", - }, + namespaceSelector: "select=this,select2=this2", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "selected-namespace", @@ -2366,10 +2357,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { }, { name: "TestSecretResourceNotInNamespaceSelector", fields: fields{ - namespaceSelector: util.Map{ - "select": "this", - "select2": "this2", - }, + namespaceSelector: "select=this,select2=this2", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "not-selected-namespace", @@ -2382,11 +2370,9 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { }, want: false, }, { - name: "TestSecretResourceInNamespaceSelectorWiledcardValue", + name: "TestSecretResourceInNamespaceSelectorKeyExists", fields: fields{ - namespaceSelector: util.Map{ - "select": "*", - }, + namespaceSelector: "select", namespace: v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "selected-namespace", @@ -2400,6 +2386,59 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { raw: testutil.GetSecret("selected-namespace", "secret", "test"), }, want: true, + }, { + name: "TestSecretResourceInNamespaceSelectorValueIn", + fields: fields{ + namespaceSelector: "select in (select1, select2, select3)", + namespace: v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "selected-namespace", + Labels: map[string]string{ + "select": "select2", + }, + }, + }, + }, + args: args{ + raw: testutil.GetSecret("selected-namespace", "secret", "test"), + }, + want: true, + }, { + name: "TestSecretResourceInNamespaceSelectorKeyDoesNotExist", + fields: fields{ + namespaceSelector: "!select2", + namespace: v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "selected-namespace", + Labels: map[string]string{ + "select": "this", + }, + }, + }, + }, + args: args{ + raw: testutil.GetSecret("selected-namespace", "secret", "test"), + }, + want: true, + }, { + name: "TestSecretResourceInNamespaceSelectorMultipleConditions", + fields: fields{ + namespaceSelector: "select,select2=this2,select3!=this4", + namespace: v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "selected-namespace", + Labels: map[string]string{ + "select": "this", + "select2": "this2", + "select3": "this3", + }, + }, + }, + }, + args: args{ + raw: testutil.GetSecret("selected-namespace", "secret", "test"), + }, + want: true, }, } @@ -2418,9 +2457,21 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { namespaceSelector: tt.fields.namespaceSelector, } - if got := c.resourceInNamespaceSelector(tt.args.raw); got != tt.want { + listOptions := metav1.ListOptions{} + listOptions.LabelSelector = tt.fields.namespaceSelector + namespaces, _ := fakeClient.CoreV1().Namespaces().List(context.Background(), listOptions) + + for _, ns := range namespaces.Items { + c.addSelectedNamespaceToCache(ns) + } + + if got := c.resourceInSelectedNamespaces(tt.args.raw); got != tt.want { t.Errorf("Controller.resourceInNamespaceSelector() = %v, want %v", got, tt.want) } + + for _, ns := range namespaces.Items { + c.removeSelectedNamespaceFromCache(ns) + } }) } } diff --git a/internal/pkg/leadership/leadership_test.go b/internal/pkg/leadership/leadership_test.go index 99f638c2..8563803e 100644 --- a/internal/pkg/leadership/leadership_test.go +++ b/internal/pkg/leadership/leadership_test.go @@ -119,7 +119,7 @@ func TestRunLeaderElectionWithControllers(t *testing.T) { t.Logf("Creating controller") var controllers []*controller.Controller for k := range kube.ResourceMap { - c, err := controller.NewController(testutil.Clients.KubernetesClient, k, testutil.Namespace, []string{}, map[string]string{}, metrics.NewCollectors()) + c, err := controller.NewController(testutil.Clients.KubernetesClient, k, testutil.Namespace, []string{}, map[string]string{}, map[string]string{}, metrics.NewCollectors()) if err != nil { logrus.Fatalf("%s", err) } From 2b619a92438ce30d1d2bebd100476a7b99c5e956 Mon Sep 17 00:00:00 2001 From: ctrought <65360454+ctrought@users.noreply.github.com> Date: Sun, 2 Apr 2023 01:31:53 -0400 Subject: [PATCH 3/8] fix: namespace list/watch permission when using namespaceSelectors Signed-off-by: Craig Trought --- .../kubernetes/chart/reloader/templates/clusterrole.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml b/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml index ed560338..c3420553 100644 --- a/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml +++ b/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml @@ -38,6 +38,8 @@ rules: - namespaces verbs: - get + - list + - watch {{- end }} {{- if and (.Capabilities.APIVersions.Has "apps.openshift.io/v1") (.Values.reloader.isOpenshift) }} - apiGroups: From f00448dfbd56c689ed56985a4952a75fd6155196 Mon Sep 17 00:00:00 2001 From: ctrought <65360454+ctrought@users.noreply.github.com> Date: Sun, 2 Apr 2023 01:47:25 -0400 Subject: [PATCH 4/8] docs: cleanup labelSelector documentation Signed-off-by: Craig Trought --- README.md | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 9e390b52..88d7e31b 100644 --- a/README.md +++ b/README.md @@ -182,14 +182,16 @@ Reloader can be configured to ignore the resources `secrets` and `configmaps` by | --resources-to-ignore=configMaps | To ignore configMaps | | --resources-to-ignore=secrets | To ignore secrets | -`Note`: At one time only one of these resource can be ignored, trying to do it will cause error in Reloader. Workaround for ignoring both resources is by scaling down the reloader pods to `0`. +**Note:** At one time only one of these resource can be ignored, trying to do it will cause error in Reloader. Workaround for ignoring both resources is by scaling down the reloader pods to `0`. -Reloader can be configured to watch only secrets/configmaps labeled with (one or more) labels of your choosing by using the `--resource-label-selector` parameter. Supported operators are `!, in, notin, ==, =, !=`, if no operator is found the 'exists' operator is inferred (ie. key only). The `:` delimited key value mappings are deprecated and if provided will be translated to key=value. Likewise, if a wildcard is provided (e.g. `key:*`) it will be translated to just the standalone `key` which checks for key existence. +Reloader can be configured to only watch secrets/configmaps with one or more labels using the `--resource-label-selector` parameter. Supported operators are `!, in, notin, ==, =, !=`, if no operator is found the 'exists' operator is inferred (ie. key only). Additional examples of these selectors can be found in the [Kubernetes Docs](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors). -These can be combined together, for example: +**Note:** The old `:` delimited key value mappings are deprecated and if provided will be translated to `key=value`. Likewise, if a wildcard value is provided (e.g. `key:*`) it will be translated to the standalone `key` which checks for key existence. + +These selectors can be combined together, for example with: ``` ---resource-label-selector=reloader=enabled,key-exists,another-label in (value1, value2, value3) +--resource-label-selector=reloader=enabled,key-exists,another-label in (value1,value2,value3) ``` Only configmaps or secrets labeled like the following will be watched: @@ -206,17 +208,16 @@ metadata: ... ``` -If you want to select namespace only by the key of the label use ```*``` as the value. -For example, for ```--namespace-selector=select-this:*``` all namespaces with label-key "select-this" will be selected regardless of the labels value +Reloader can be configured to only watch namespaces labeled with one or more labels using the `--namespace-selector` parameter. Supported operators are `!, in, notin, ==, =, !=`, if no operator is found the 'exists' operator is inferred (ie. key only). Additional examples of these selectors can be found in the (Kubernetes Docs)[https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors]. -Reloader can be configured to watch only namespaces labeled with (one or more) labels of your choosing by using the `--namespace-selector` parameter. Reloader can be configured to watch only secrets/configmaps labeled with (one or more) labels of your choosing by using the `--resource-label-selector` parameter. Supported operators are `!, in, notin, ==, =, !=`, if no operator is found the 'exists' operator is inferred (ie. key only). The `:` delimited key value mappings are deprecated and if provided will be translated to key=value. Likewise, if a wildcard is provided (e.g. `key:*`) it will be translated to just the standalone `key` which checks for key existence. +**Note:** The old `:` delimited key value mappings are deprecated and if provided will be translated to `key=value`. Likewise, if a wildcard value is provided (e.g. `key:*`) it will be translated to the standalone `key` which checks for key existence. -These can be combined together, for example: +These selectors can be combined together, for example with: ``` ---namespace-selector=reloader:enabled,test:true +--namespace-selector=reloader=enabled,test=true ``` -Only namespaces labeled like the following namespace YAML will be watched: +Only namespaces labeled as below would be watched and eligible for reloads: ```yaml kind: Namespace apiVersion: v1 @@ -227,8 +228,6 @@ metadata: test: true ... ``` -If you want to select namespace only by the key of the label use ```*``` as the value. -For example, for ```--namespace-selector=select-this:*``` all namespaces with label-key "select-this" will be selected regardless of the labels value ### Vanilla kustomize @@ -279,20 +278,22 @@ Reloader can be configured to ignore the resources `secrets` and `configmaps` by | ignoreSecrets | To ignore secrets. Valid value are either `true` or `false` | boolean | | ignoreConfigMaps | To ignore configMaps. Valid value are either `true` or `false` | boolean | -`Note`: At one time only one of these resource can be ignored, trying to do it will cause error in helm template compilation. +**Note:** At one time only one of these resource can be ignored, trying to do it will cause error in helm template compilation. -Reloader can be configured to watch only namespaces labeled with (one or more) labels of your choosing by using the `namespaceSelector` parameter +Reloader can be configured to only watch namespaces labeled with one or more labels using the `namespaceSelector` parameter | Parameter | Description | Type | | ---------------- | ---------------------------------------------------------------------------------- | ------- | | namespaceSelector | list of comma separated label selectors, if mulitple are provided they are ANDed | string | -Reloader can be configured to watch only configmaps/secrets labeled with (one or more) labels of your choosing by using the `resourceLabelSelector` parameter +Reloader can be configured to only watch configmaps/secrets labeled with one or more labels using the `resourceLabelSelector` parameter | Parameter | Description | Type | | ---------------------- | ---------------------------------------------------------------------------------- | ------- | | resourceLabelSelector | list of comma separated label selectors, if mulitple are provided they are ANDed | string | +**Note:** Both `namespaceSelector` & `resourceLabelSelector` can be used together. If they are then both conditions must be met for the configmap or secret to be eligible to trigger reload events. (e.g. If a configMap matches `resourceLabelSelector` but `namespaceSelector` does not match the namespace the configmap is in, it will be ignored) + You can also set the log format of Reloader to json by setting `logFormat` to `json` in values.yaml and apply the chart You can enable to scrape Reloader's Prometheus metrics by setting `serviceMonitor.enabled` or `podMonitor.enabled` to `true` in values.yaml file. Service monitor will be removed in future releases of reloader in favour of Pod monitor. @@ -352,7 +353,7 @@ PRs are welcome. In general, we follow the "fork-and-pull" Git workflow. 4. **Push** your work back up to your fork 5. Submit a **Pull request** so that we can review your changes -NOTE: Be sure to merge the latest from "upstream" before making a pull request! +**NOTE:** Be sure to merge the latest from "upstream" before making a pull request! ## Changelog From 8ed0899ff3cd2915d587d6b82775fbcc3f3023a3 Mon Sep 17 00:00:00 2001 From: ctrought <65360454+ctrought@users.noreply.github.com> Date: Sun, 2 Apr 2023 15:13:07 -0400 Subject: [PATCH 5/8] fix: use apimachinery labelSelector parser to align with string implementation, supports != operator Signed-off-by: Craig Trought --- internal/pkg/cmd/reloader.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index 0158be14..874e776b 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -19,6 +19,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/util" "github.com/stakater/Reloader/pkg/kube" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) // NewReloaderCommand starts the reloader controller @@ -223,7 +224,7 @@ func getNamespaceLabelSelector(cmd *cobra.Command) (string, error) { } namespaceLabelSelector := strings.Join(slice[:], ",") - _, err = v1.ParseToLabelSelector(namespaceLabelSelector) + _, err = labels.Parse(namespaceLabelSelector) if err != nil { logrus.Fatal(err) } @@ -257,7 +258,7 @@ func getResourceLabelSelector(cmd *cobra.Command) (string, error) { } resourceLabelSelector := strings.Join(slice[:], ",") - _, err = v1.ParseToLabelSelector(resourceLabelSelector) + _, err = labels.Parse(resourceLabelSelector) if err != nil { logrus.Fatal(err) } From 8e8ce5131387a296a30a2132531925191598ba18 Mon Sep 17 00:00:00 2001 From: ctrought <65360454+ctrought@users.noreply.github.com> Date: Sun, 2 Apr 2023 15:48:12 -0400 Subject: [PATCH 6/8] docs: update label selector usage Signed-off-by: Craig Trought --- deployments/kubernetes/chart/reloader/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/kubernetes/chart/reloader/values.yaml b/deployments/kubernetes/chart/reloader/values.yaml index 5f9ab209..e4ac6a1d 100644 --- a/deployments/kubernetes/chart/reloader/values.yaml +++ b/deployments/kubernetes/chart/reloader/values.yaml @@ -20,8 +20,8 @@ reloader: syncAfterRestart: false reloadStrategy: default # Set to default, env-vars or annotations ignoreNamespaces: "" # Comma separated list of namespaces to ignore - namespaceSelector: "" # Comma separated list of 'key:value' labels for namespaces selection - resourceLabelSelector: "" # Comma separated list of 'key:value' labels for configmap/secret selection + namespaceSelector: "" # Comma separated list of k8s label selectors for namespaces selection + resourceLabelSelector: "" # Comma separated list of k8s label selectors for configmap/secret selection logFormat: "" #json watchGlobally: true # Set to true to enable leadership election allowing you to run multiple replicas From 24e794bb385bb7c9b46fee604e39593d3aa74e3e Mon Sep 17 00:00:00 2001 From: ctrought <65360454+ctrought@users.noreply.github.com> Date: Sun, 2 Apr 2023 16:47:17 -0400 Subject: [PATCH 7/8] fix: skip controller creation for namespace in test Signed-off-by: Craig Trought --- internal/pkg/controller/controller_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/pkg/controller/controller_test.go b/internal/pkg/controller/controller_test.go index 62a9370e..509f7f3e 100644 --- a/internal/pkg/controller/controller_test.go +++ b/internal/pkg/controller/controller_test.go @@ -45,6 +45,9 @@ func TestMain(m *testing.M) { logrus.Infof("Creating controller") for k := range kube.ResourceMap { + if k == "namespaces" { + continue + } c, err := NewController(clients.KubernetesClient, k, namespace, []string{}, "", "", collectors) if err != nil { logrus.Fatalf("%s", err) From 9ac351c21940a5ff071539c9f2d287eefb483400 Mon Sep 17 00:00:00 2001 From: ctrought <65360454+ctrought@users.noreply.github.com> Date: Sun, 2 Apr 2023 16:47:54 -0400 Subject: [PATCH 8/8] fix: controller label selectors as string Signed-off-by: Craig Trought --- internal/pkg/leadership/leadership_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/leadership/leadership_test.go b/internal/pkg/leadership/leadership_test.go index 8563803e..085b7509 100644 --- a/internal/pkg/leadership/leadership_test.go +++ b/internal/pkg/leadership/leadership_test.go @@ -119,7 +119,7 @@ func TestRunLeaderElectionWithControllers(t *testing.T) { t.Logf("Creating controller") var controllers []*controller.Controller for k := range kube.ResourceMap { - c, err := controller.NewController(testutil.Clients.KubernetesClient, k, testutil.Namespace, []string{}, map[string]string{}, map[string]string{}, metrics.NewCollectors()) + c, err := controller.NewController(testutil.Clients.KubernetesClient, k, testutil.Namespace, []string{}, "", "", metrics.NewCollectors()) if err != nil { logrus.Fatalf("%s", err) }