support labelselector set (#73)

Signed-off-by: ldpliu <daliu@redhat.com>

deploy/hub/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml

@@ -120,12 +120,43 @@ spec:
                   default:
                     selectorType: LegacyClusterSetLabel
                   properties:
+                    labelSelector:
+                      description: LabelSelector define the general labelSelector which clusterset will use to select target managedClusters
+                      type: object
+                      properties:
+                        matchExpressions:
+                          description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                          type: array
+                          items:
+                            description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                            type: object
+                            required:
+                              - key
+                              - operator
+                            properties:
+                              key:
+                                description: key is the label key that the selector applies to.
+                                type: string
+                              operator:
+                                description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                                type: string
+                              values:
+                                description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                                type: array
+                                items:
+                                  type: string
+                        matchLabels:
+                          description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                          type: object
+                          additionalProperties:
+                            type: string
                     selectorType:
-                      description: SelectorType could only be "LegacyClusterSetLabel" now, will support more SelectorType later "LegacyClusterSetLabel" means to use label "cluster.open-cluster-management.io/clusterset:<ManagedClusterSet Name>"" to select target clusters.
+                      description: SelectorType could only be "LegacyClusterSetLabel" or "LabelSelector" "LegacyClusterSetLabel" means to use label "cluster.open-cluster-management.io/clusterset:<ManagedClusterSet Name>"" to select target clusters. "LabelSelector" means use labelSelector to select target managedClusters
                       type: string
                       default: LegacyClusterSetLabel
                       enum:
                         - LegacyClusterSetLabel
+                        - LabelSelector
             status:
               description: Status represents the current status of the ManagedClusterSet
               type: object

deploy/hub/0000_01_clusters.open-cluster-management.io_managedclustersetbindings.crd.yaml

@@ -63,8 +63,58 @@ spec:
                   description: ClusterSet is the name of the ManagedClusterSet to bind. It must match the instance name of the ManagedClusterSetBinding and cannot change once created. User is allowed to set this field if they have an RBAC rule to CREATE on the virtual subresource of managedclustersets/bind.
                   type: string
                   minLength: 1
+            status:
+              description: Status represents the current status of the ManagedClusterSetBinding
+              type: object
+              properties:
+                conditions:
+                  description: Conditions contains the different condition statuses for this ManagedClusterSetBinding.
+                  type: array
+                  items:
+                    description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
+                    type: object
+                    required:
+                      - lastTransitionTime
+                      - message
+                      - reason
+                      - status
+                      - type
+                    properties:
+                      lastTransitionTime:
+                        description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                        type: string
+                        format: date-time
+                      message:
+                        description: message is a human readable message indicating details about the transition. This may be an empty string.
+                        type: string
+                        maxLength: 32768
+                      observedGeneration:
+                        description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
+                        type: integer
+                        format: int64
+                        minimum: 0
+                      reason:
+                        description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.
+                        type: string
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      status:
+                        description: status of the condition, one of True, False, Unknown.
+                        type: string
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
+                      type:
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                        type: string
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
       served: true
       storage: true
+      subresources:
+        status: {}
 status:
   acceptedNames:
     kind: ""

go.mod

@@ -18,7 +18,7 @@ require (
 	k8s.io/component-base v0.23.5
 	k8s.io/klog/v2 v2.60.1
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
-	open-cluster-management.io/api v0.7.0
+	open-cluster-management.io/api v0.7.1-0.20220530034043-2b50efdde1de
 	sigs.k8s.io/controller-runtime v0.11.2
 )
 

go.sum

@@ -1026,8 +1026,8 @@ k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc=
 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-open-cluster-management.io/api v0.7.0 h1:Xt1tRCwt+wrhtCOEQ6g+7sFvIkMjffWnn5PSUSoKJcc=
-open-cluster-management.io/api v0.7.0/go.mod h1:Wg7YOcVNxsNDj2G8ViWTD/utCfb9cZc9MpNb4fKlXSs=
+open-cluster-management.io/api v0.7.1-0.20220530034043-2b50efdde1de h1:xxlXg3eM5DXa7ezj7J++CggxyaHjYuC67o28w6cFMiM=
+open-cluster-management.io/api v0.7.1-0.20220530034043-2b50efdde1de/go.mod h1:QKW4hRonyzoXavBX8XK/Rljo4PYEKKt/IOShuLv49XI=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

pkg/controllers/scheduling/scheduling_controller.go

@@ -352,58 +352,37 @@ func (c *schedulingController) getAvailableClusters(clusterSetNames []string) ([
 	if len(clusterSetNames) == 0 {
 		return nil, nil
 	}
+	//all available clusters
+	availableClusters := map[string]*clusterapiv1.ManagedCluster{}
 
-	// filter avilable cluserset label and values
-	avilableClusterSetsLabels := map[string]sets.String{}
 	for _, name := range clusterSetNames {
 		// ignore clusterset if failed to get
 		clusterSet, err := c.clusterSetLister.Get(name)
-		if err != nil {
+		if errors.IsNotFound(err) {
 			continue
 		}
-
-		// clusterset ClusterSelector type is empty or LegacyClusterSetLabel is treated as avaliable
-		selectorType := clusterSet.Spec.ClusterSelector.SelectorType
-		if len(selectorType) == 0 || selectorType == clusterapiv1beta1.LegacyClusterSetLabel {
-			if avilableClusterSetsLabels[clusterSetLabel] == nil {
-				avilableClusterSetsLabels[clusterSetLabel] = sets.NewString()
-			}
-			// store label and values
-			avilableClusterSetsLabels[clusterSetLabel].Insert(name)
-		}
-	}
-
-	// list all avilable clusters
-	avilableClusters := map[string]*clusterapiv1.ManagedCluster{}
-	for label, vals := range avilableClusterSetsLabels {
-		requirement, err := labels.NewRequirement(label, selection.In, vals.List())
 		if err != nil {
-			klog.Warning(err)
-			continue
+			return nil, err
 		}
-
-		labelSelector := labels.NewSelector().Add(*requirement)
-		clusters, err := c.clusterLister.List(labelSelector)
+		clusters, err := clusterapiv1beta1.GetClustersFromClusterSet(clusterSet, c.clusterLister)
 		if err != nil {
-			klog.Warning(err)
-			continue
+			return nil, fmt.Errorf("failed to get clusterset: %v, clusters, Error: %v", clusterSet.Name, err)
 		}
 		for i := range clusters {
-			avilableClusters[clusters[i].Name] = clusters[i]
+			availableClusters[clusters[i].Name] = clusters[i]
 		}
 	}
 
-	if len(avilableClusters) == 0 {
+	if len(availableClusters) == 0 {
 		return nil, nil
 	}
 
 	result := []*clusterapiv1.ManagedCluster{}
-	for _, c := range avilableClusters {
+	for _, c := range availableClusters {
 		result = append(result, c)
 	}
 
 	return result, nil
-
 }
 
 // updateStatus updates the status of the placement according to intermediate scheduling data.

pkg/controllers/scheduling/scheduling_controller_test.go

@@ -369,19 +369,64 @@ func TestGetAvailableClusters(t *testing.T) {
 			expectedClusterNames: []string{"cluster1"},
 		},
 		{
-			name:            "clusterset has valid ClusterSelector",
+			name:            "clusterset has default ClusterSelector",
 			clusterSetNames: []string{"clusterset1"},
 			initObjs: []runtime.Object{
-				testinghelpers.NewClusterSet("clusterset1").WithClusterSelector(clusterapiv1beta1.LegacyClusterSetLabel).Build(),
+				testinghelpers.NewClusterSet("clusterset1").WithClusterSelector(clusterapiv1beta1.ManagedClusterSelector{}).Build(),
 				testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(),
 			},
 			expectedClusterNames: []string{"cluster1"},
 		},
+		{
+			name:            "clusterset has Legacy set label ClusterSelector",
+			clusterSetNames: []string{"clusterset1"},
+			initObjs: []runtime.Object{
+				testinghelpers.NewClusterSet("clusterset1").WithClusterSelector(clusterapiv1beta1.ManagedClusterSelector{
+					SelectorType: clusterapiv1beta1.LegacyClusterSetLabel,
+				}).Build(),
+				testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(),
+			},
+			expectedClusterNames: []string{"cluster1"},
+		},
+		{
+			name:            "clusterset has labelSelector type ClusterSelector",
+			clusterSetNames: []string{"clusterset1"},
+			initObjs: []runtime.Object{
+				testinghelpers.NewClusterSet("clusterset1").WithClusterSelector(clusterapiv1beta1.ManagedClusterSelector{
+					SelectorType: clusterapiv1beta1.LabelSelector,
+					LabelSelector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{
+							"vendor": "openShift",
+						},
+					},
+				}).Build(),
+				testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(),
+				testinghelpers.NewManagedCluster("cluster2").WithLabel("vendor", "openShift").Build(),
+			},
+			expectedClusterNames: []string{"cluster2"},
+		},
+		{
+			name:            "clusterset has labelSelector type ClusterSelector(select everything)",
+			clusterSetNames: []string{"clusterset1"},
+			initObjs: []runtime.Object{
+				testinghelpers.NewClusterSet("clusterset1").WithClusterSelector(clusterapiv1beta1.ManagedClusterSelector{
+					SelectorType:  clusterapiv1beta1.LabelSelector,
+					LabelSelector: &metav1.LabelSelector{},
+				}).Build(),
+				testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(),
+				testinghelpers.NewManagedCluster("cluster2").Build(),
+			},
+			expectedClusterNames: []string{"cluster1", "cluster2"},
+		},
 		{
 			name:            "clusterset has invalid ClusterSelector",
 			clusterSetNames: []string{"clusterset1"},
 			initObjs: []runtime.Object{
-				testinghelpers.NewClusterSet("clusterset1").WithClusterSelector("FutureClusterSetLabel").Build(),
+				testinghelpers.NewClusterSet("clusterset1").WithClusterSelector(
+					clusterapiv1beta1.ManagedClusterSelector{
+						SelectorType: "errorType",
+					},
+				).Build(),
 				testinghelpers.NewManagedCluster("cluster1").WithLabel(clusterSetLabel, "clusterset1").Build(),
 			},
 			expectedClusterNames: []string{},

pkg/helpers/testing/builders.go

@@ -281,8 +281,8 @@ func NewClusterSet(clusterSetName string) *managedClusterSetBuilder {
 	}
 }
 
-func (b *managedClusterSetBuilder) WithClusterSelector(SelectorType clusterapiv1beta1.SelectorType) *managedClusterSetBuilder {
-	b.clusterset.Spec.ClusterSelector.SelectorType = SelectorType
+func (b *managedClusterSetBuilder) WithClusterSelector(clusterSelector clusterapiv1beta1.ManagedClusterSelector) *managedClusterSetBuilder {
+	b.clusterset.Spec.ClusterSelector = clusterSelector
 	return b
 }
 

test/integration/placement_test.go

@@ -235,6 +235,25 @@ var _ = ginkgo.Describe("Placement", func() {
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 	}
 
+	assertCreatingLabelSelectorClusterSet := func(clusterSetName string, matchLabel map[string]string) {
+		ginkgo.By(fmt.Sprintf("Create clusterset %s", clusterSetName))
+		clusterset := &clusterapiv1beta1.ManagedClusterSet{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: clusterSetName,
+			},
+			Spec: clusterapiv1beta1.ManagedClusterSetSpec{
+				ClusterSelector: clusterapiv1beta1.ManagedClusterSelector{
+					SelectorType: clusterapiv1beta1.LabelSelector,
+					LabelSelector: &metav1.LabelSelector{
+						MatchLabels: matchLabel,
+					},
+				},
+			},
+		}
+		_, err = clusterClient.ClusterV1beta1().ManagedClusterSets().Create(context.Background(), clusterset, metav1.CreateOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	}
+
 	assertDeletingClusterSet := func(clusterSetName string) {
 		ginkgo.By(fmt.Sprintf("Delete clusterset %s", clusterSetName))
 		err = clusterClient.ClusterV1beta1().ManagedClusterSets().Delete(context.Background(), clusterSetName, metav1.DeleteOptions{})
@@ -280,6 +299,20 @@ var _ = ginkgo.Describe("Placement", func() {
 		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
 	}
 
+	assertCreatingClusterWithLabel := func(clusterName string, labels map[string]string) {
+		ginkgo.By(fmt.Sprintf("Create cluster %v", clusterName))
+		cluster := &clusterapiv1.ManagedCluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:   clusterName,
+				Labels: labels,
+			},
+		}
+
+		_, err = clusterClient.ClusterV1().ManagedClusters().Create(context.Background(), cluster, metav1.CreateOptions{})
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+	}
+
 	assertCreatingClusters := func(clusterSetName string, num int, labels ...string) {
 		ginkgo.By(fmt.Sprintf("Create %d clusters", num))
 		for i := 0; i < num; i++ {
@@ -953,6 +986,34 @@ var _ = ginkgo.Describe("Placement", func() {
 			assertPlacementStatus(placementName, 5, false)
 		})
 
+		ginkgo.It("Should re-schedule successfully once a labelselector type clusterset deleted/added", func() {
+			assertCreatingLabelSelectorClusterSet(clusterSet1Name, map[string]string{
+				"vendor": "openShift",
+			})
+			assertCreatingClusterSetBinding(clusterSet1Name)
+			assertCreatingClusterWithLabel("cluster1", map[string]string{
+				"vendor": "openShift",
+			})
+			assertCreatingClusterWithLabel("cluster2", map[string]string{
+				"vendor": "IKS",
+			})
+			assertCreatingPlacement(placementName, noc(10), 1, clusterapiv1beta1.PrioritizerPolicy{}, []clusterapiv1beta1.Toleration{})
+
+			assertNumberOfDecisions(placementName, 1)
+			assertPlacementStatus(placementName, 1, false)
+
+			ginkgo.By("Delete the clusterset")
+			assertDeletingClusterSet(clusterSet1Name)
+
+			assertNumberOfDecisions(placementName, 0)
+
+			ginkgo.By("Add the clusterset back")
+			assertCreatingLabelSelectorClusterSet(clusterSet1Name, map[string]string{
+				"vendor": "openShift",
+			})
+			assertNumberOfDecisions(placementName, 1)
+			assertPlacementStatus(placementName, 1, false)
+		})
 		ginkgo.It("Should re-schedule successfully once a clustersetbinding deleted/added", func() {
 			assertBindingClusterSet(clusterSet1Name)
 			assertCreatingClusters(clusterSet1Name, 5)

vendor/modules.txt

@@ -1060,8 +1060,8 @@ k8s.io/utils/net
 k8s.io/utils/path
 k8s.io/utils/pointer
 k8s.io/utils/trace
-# open-cluster-management.io/api v0.7.0
-## explicit; go 1.17
+# open-cluster-management.io/api v0.7.1-0.20220530034043-2b50efdde1de
+## explicit; go 1.18
 open-cluster-management.io/api/client/cluster/clientset/versioned
 open-cluster-management.io/api/client/cluster/clientset/versioned/fake
 open-cluster-management.io/api/client/cluster/clientset/versioned/scheme

vendor/open-cluster-management.io/api/client/cluster/clientset/versioned/typed/cluster/v1beta1/fake/fake_managedclustersetbinding.go

@@ -86,6 +86,18 @@ func (c *FakeManagedClusterSetBindings) Update(ctx context.Context, managedClust
 	return obj.(*v1beta1.ManagedClusterSetBinding), err
 }
 
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+func (c *FakeManagedClusterSetBindings) UpdateStatus(ctx context.Context, managedClusterSetBinding *v1beta1.ManagedClusterSetBinding, opts v1.UpdateOptions) (*v1beta1.ManagedClusterSetBinding, error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewUpdateSubresourceAction(managedclustersetbindingsResource, "status", c.ns, managedClusterSetBinding), &v1beta1.ManagedClusterSetBinding{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.ManagedClusterSetBinding), err
+}
+
 // Delete takes name of the managedClusterSetBinding and deletes it. Returns an error if one occurs.
 func (c *FakeManagedClusterSetBindings) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
 	_, err := c.Fake.

vendor/open-cluster-management.io/api/client/cluster/clientset/versioned/typed/cluster/v1beta1/managedclustersetbinding.go

@@ -24,6 +24,7 @@ type ManagedClusterSetBindingsGetter interface {
 type ManagedClusterSetBindingInterface interface {
 	Create(ctx context.Context, managedClusterSetBinding *v1beta1.ManagedClusterSetBinding, opts v1.CreateOptions) (*v1beta1.ManagedClusterSetBinding, error)
 	Update(ctx context.Context, managedClusterSetBinding *v1beta1.ManagedClusterSetBinding, opts v1.UpdateOptions) (*v1beta1.ManagedClusterSetBinding, error)
+	UpdateStatus(ctx context.Context, managedClusterSetBinding *v1beta1.ManagedClusterSetBinding, opts v1.UpdateOptions) (*v1beta1.ManagedClusterSetBinding, error)
 	Delete(ctx context.Context, name string, opts v1.DeleteOptions) error
 	DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error
 	Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.ManagedClusterSetBinding, error)
@@ -119,6 +120,22 @@ func (c *managedClusterSetBindings) Update(ctx context.Context, managedClusterSe
 	return
 }
 
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+func (c *managedClusterSetBindings) UpdateStatus(ctx context.Context, managedClusterSetBinding *v1beta1.ManagedClusterSetBinding, opts v1.UpdateOptions) (result *v1beta1.ManagedClusterSetBinding, err error) {
+	result = &v1beta1.ManagedClusterSetBinding{}
+	err = c.client.Put().
+		Namespace(c.ns).
+		Resource("managedclustersetbindings").
+		Name(managedClusterSetBinding.Name).
+		SubResource("status").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(managedClusterSetBinding).
+		Do(ctx).
+		Into(result)
+	return
+}
+
 // Delete takes name of the managedClusterSetBinding and deletes it. Returns an error if one occurs.
 func (c *managedClusterSetBindings) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
 	return c.client.Delete().

vendor/open-cluster-management.io/api/cluster/v1beta1/0000_00_clusters.open-cluster-management.io_managedclustersets.crd.yaml

@@ -120,12 +120,43 @@ spec:
                   default:
                     selectorType: LegacyClusterSetLabel
                   properties:
+                    labelSelector:
+                      description: LabelSelector define the general labelSelector which clusterset will use to select target managedClusters
+                      type: object
+                      properties:
+                        matchExpressions:
+                          description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                          type: array
+                          items:
+                            description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                            type: object
+                            required:
+                              - key
+                              - operator
+                            properties:
+                              key:
+                                description: key is the label key that the selector applies to.
+                                type: string
+                              operator:
+                                description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                                type: string
+                              values:
+                                description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                                type: array
+                                items:
+                                  type: string
+                        matchLabels:
+                          description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                          type: object
+                          additionalProperties:
+                            type: string
                     selectorType:
-                      description: SelectorType could only be "LegacyClusterSetLabel" now, will support more SelectorType later "LegacyClusterSetLabel" means to use label "cluster.open-cluster-management.io/clusterset:<ManagedClusterSet Name>"" to select target clusters.
+                      description: SelectorType could only be "LegacyClusterSetLabel" or "LabelSelector" "LegacyClusterSetLabel" means to use label "cluster.open-cluster-management.io/clusterset:<ManagedClusterSet Name>"" to select target clusters. "LabelSelector" means use labelSelector to select target managedClusters
                       type: string
                       default: LegacyClusterSetLabel
                       enum:
                         - LegacyClusterSetLabel
+                        - LabelSelector
             status:
               description: Status represents the current status of the ManagedClusterSet
               type: object

vendor/open-cluster-management.io/api/cluster/v1beta1/0000_01_clusters.open-cluster-management.io_managedclustersetbindings.crd.yaml

@@ -63,8 +63,58 @@ spec:
                   description: ClusterSet is the name of the ManagedClusterSet to bind. It must match the instance name of the ManagedClusterSetBinding and cannot change once created. User is allowed to set this field if they have an RBAC rule to CREATE on the virtual subresource of managedclustersets/bind.
                   type: string
                   minLength: 1
+            status:
+              description: Status represents the current status of the ManagedClusterSetBinding
+              type: object
+              properties:
+                conditions:
+                  description: Conditions contains the different condition statuses for this ManagedClusterSetBinding.
+                  type: array
+                  items:
+                    description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions.  For example, type FooStatus struct{     // Represents the observations of a foo's current state.     // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type     // +patchStrategy=merge     // +listType=map     // +listMapKey=type     Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n     // other fields }"
+                    type: object
+                    required:
+                      - lastTransitionTime
+                      - message
+                      - reason
+                      - status
+                      - type
+                    properties:
+                      lastTransitionTime:
+                        description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                        type: string
+                        format: date-time
+                      message:
+                        description: message is a human readable message indicating details about the transition. This may be an empty string.
+                        type: string
+                        maxLength: 32768
+                      observedGeneration:
+                        description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
+                        type: integer
+                        format: int64
+                        minimum: 0
+                      reason:
+                        description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.
+                        type: string
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      status:
+                        description: status of the condition, one of True, False, Unknown.
+                        type: string
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
+                      type:
+                        description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                        type: string
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
       served: true
       storage: true
+      subresources:
+        status: {}
 status:
   acceptedNames:
     kind: ""

vendor/open-cluster-management.io/api/cluster/v1beta1/helpers.go

@@ -0,0 +1,110 @@
+package v1beta1
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	v1 "open-cluster-management.io/api/cluster/v1"
+)
+
+type ManagedClustersGetter interface {
+	List(selector labels.Selector) (ret []*v1.ManagedCluster, err error)
+}
+
+type ManagedClusterSetsGetter interface {
+	List(selector labels.Selector) (ret []*ManagedClusterSet, err error)
+}
+
+type ManagedClusterSetBindingsGetter interface {
+	List(namespace string, selector labels.Selector) (ret []*ManagedClusterSetBinding, err error)
+}
+
+// GetClustersFromClusterSet return the ManagedClusterSet's managedClusters
+func GetClustersFromClusterSet(clusterSet *ManagedClusterSet,
+	clustersGetter ManagedClustersGetter) ([]*v1.ManagedCluster, error) {
+	var clusters []*v1.ManagedCluster
+
+	if clusterSet == nil {
+		return nil, nil
+	}
+
+	clusterSelector, err := BuildClusterSelector(clusterSet)
+	if err != nil {
+		return nil, err
+	}
+	if clusterSelector == nil {
+		return nil, fmt.Errorf("failed to build ClusterSelector with clusterSet: %v", clusterSet)
+	}
+	clusters, err = clustersGetter.List(clusterSelector)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list ManagedClusters: %w", err)
+	}
+	return clusters, nil
+}
+
+// GetClusterSetsOfClusterByCluster return the managedClusterSets of a managedCluster
+func GetClusterSetsOfCluster(cluster *v1.ManagedCluster,
+	clusterSetsGetter ManagedClusterSetsGetter) ([]*ManagedClusterSet, error) {
+	var returnClusterSets []*ManagedClusterSet
+
+	if cluster == nil {
+		return nil, nil
+	}
+
+	allClusterSets, err := clusterSetsGetter.List(labels.Everything())
+	if err != nil {
+		return nil, err
+	}
+	for _, clusterSet := range allClusterSets {
+		clusterSelector, err := BuildClusterSelector(clusterSet)
+		if err != nil {
+			return nil, err
+		}
+		if clusterSelector == nil {
+			return nil, fmt.Errorf("failed to build ClusterSelector with clusterSet: %v", clusterSet)
+		}
+		if clusterSelector.Matches(labels.Set(cluster.Labels)) {
+			returnClusterSets = append(returnClusterSets, clusterSet)
+		}
+	}
+	return returnClusterSets, nil
+}
+
+func BuildClusterSelector(clusterSet *ManagedClusterSet) (labels.Selector, error) {
+	if clusterSet == nil {
+		return nil, nil
+	}
+	selectorType := clusterSet.Spec.ClusterSelector.SelectorType
+
+	switch selectorType {
+	case "", LegacyClusterSetLabel:
+		return labels.SelectorFromSet(labels.Set{
+			ClusterSetLabel: clusterSet.Name,
+		}), nil
+	case LabelSelector:
+		return metav1.LabelSelectorAsSelector(clusterSet.Spec.ClusterSelector.LabelSelector)
+	default:
+		return nil, fmt.Errorf("selectorType is not right: %s", clusterSet.Spec.ClusterSelector.SelectorType)
+	}
+}
+
+// GetBoundManagedClusterSetBindings returns all bindings that are bounded to clustersets in the given namespace.
+func GetBoundManagedClusterSetBindings(namespace string,
+	clusterSetBindingsGetter ManagedClusterSetBindingsGetter) ([]*ManagedClusterSetBinding, error) {
+	// get all clusterset bindings under the namespace
+	bindings, err := clusterSetBindingsGetter.List(namespace, labels.Everything())
+	if err != nil {
+		return nil, err
+	}
+
+	boundBindings := []*ManagedClusterSetBinding{}
+	for _, binding := range bindings {
+		if meta.IsStatusConditionTrue(binding.Status.Conditions, ClusterSetBindingBoundType) {
+			boundBindings = append(boundBindings, binding)
+		}
+	}
+
+	return boundBindings, nil
+}

vendor/open-cluster-management.io/api/cluster/v1beta1/types_managedclusterset.go

@@ -4,6 +4,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+//LegacyClusterSetLabel LabelKey
+const ClusterSetLabel = "cluster.open-cluster-management.io/clusterset"
+
 // +genclient
 // +genclient:nonNamespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -47,18 +50,26 @@ type ManagedClusterSetSpec struct {
 
 // ManagedClusterSelector represents a selector of ManagedClusters
 type ManagedClusterSelector struct {
-	// SelectorType could only be "LegacyClusterSetLabel" now, will support more SelectorType later
+	// SelectorType could only be "LegacyClusterSetLabel" or "LabelSelector"
 	// "LegacyClusterSetLabel" means to use label "cluster.open-cluster-management.io/clusterset:<ManagedClusterSet Name>"" to select target clusters.
-	// +kubebuilder:validation:Enum=LegacyClusterSetLabel
+	// "LabelSelector" means use labelSelector to select target managedClusters
+	// +kubebuilder:validation:Enum=LegacyClusterSetLabel;LabelSelector
 	// +kubebuilder:default:=LegacyClusterSetLabel
 	// +required
 	SelectorType SelectorType `json:"selectorType,omitempty"`
+
+	// LabelSelector define the general labelSelector which clusterset will use to select target managedClusters
+	// +optional
+	LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"`
 }
 
 type SelectorType string
 
 const (
+	// "LegacyClusterSetLabel" means to use label "cluster.open-cluster-management.io/clusterset:<ManagedClusterSet Name>"" to select target clusters.
 	LegacyClusterSetLabel SelectorType = "LegacyClusterSetLabel"
+	// "LabelSelector" means use labelSelector to select target managedClusters
+	LabelSelector SelectorType = "LabelSelector"
 )
 
 // ManagedClusterSetStatus represents the current status of the ManagedClusterSet.

vendor/open-cluster-management.io/api/cluster/v1beta1/types_managedclustersetbinding.go

@@ -6,6 +6,7 @@ import (
 
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +kubebuilder:subresource:status
 // +kubebuilder:resource:scope="Namespaced",shortName={"mclsetbinding","mclsetbindings"}
 // +kubebuilder:storageversion
 
@@ -21,6 +22,10 @@ type ManagedClusterSetBinding struct {
 
 	// Spec defines the attributes of ManagedClusterSetBinding.
 	Spec ManagedClusterSetBindingSpec `json:"spec"`
+
+	// Status represents the current status of the ManagedClusterSetBinding
+	// +optional
+	Status ManagedClusterSetBindingStatus `json:"status,omitempty"`
 }
 
 // ManagedClusterSetBindingSpec defines the attributes of ManagedClusterSetBinding.
@@ -33,6 +38,18 @@ type ManagedClusterSetBindingSpec struct {
 	ClusterSet string `json:"clusterSet"`
 }
 
+const (
+	// ClusterSetBindingBoundType is a condition type of clustersetbinding representing
+	// whether the ClusterSetBinding is bound to a clusterset.
+	ClusterSetBindingBoundType = "Bound"
+)
+
+// ManagedClusterSetBindingStatus represents the current status of the ManagedClusterSetBinding.
+type ManagedClusterSetBindingStatus struct {
+	// Conditions contains the different condition statuses for this ManagedClusterSetBinding.
+	Conditions []metav1.Condition `json:"conditions"`
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // ManagedClusterSetBindingList is a collection of ManagedClusterSetBinding.

vendor/open-cluster-management.io/api/cluster/v1beta1/zz_generated.deepcopy.go

@@ -103,6 +103,11 @@ func (in *ClusterSelector) DeepCopy() *ClusterSelector {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ManagedClusterSelector) DeepCopyInto(out *ManagedClusterSelector) {
 	*out = *in
+	if in.LabelSelector != nil {
+		in, out := &in.LabelSelector, &out.LabelSelector
+		*out = new(v1.LabelSelector)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -121,7 +126,7 @@ func (in *ManagedClusterSet) DeepCopyInto(out *ManagedClusterSet) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	in.Status.DeepCopyInto(&out.Status)
 	return
 }
@@ -150,6 +155,7 @@ func (in *ManagedClusterSetBinding) DeepCopyInto(out *ManagedClusterSetBinding)
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
 	out.Spec = in.Spec
+	in.Status.DeepCopyInto(&out.Status)
 	return
 }
 
@@ -220,6 +226,29 @@ func (in *ManagedClusterSetBindingSpec) DeepCopy() *ManagedClusterSetBindingSpec
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ManagedClusterSetBindingStatus) DeepCopyInto(out *ManagedClusterSetBindingStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]v1.Condition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedClusterSetBindingStatus.
+func (in *ManagedClusterSetBindingStatus) DeepCopy() *ManagedClusterSetBindingStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ManagedClusterSetBindingStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ManagedClusterSetList) DeepCopyInto(out *ManagedClusterSetList) {
 	*out = *in
@@ -256,7 +285,7 @@ func (in *ManagedClusterSetList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ManagedClusterSetSpec) DeepCopyInto(out *ManagedClusterSetSpec) {
 	*out = *in
-	out.ClusterSelector = in.ClusterSelector
+	in.ClusterSelector.DeepCopyInto(&out.ClusterSelector)
 	return
 }
 

vendor/open-cluster-management.io/api/cluster/v1beta1/zz_generated.swagger_doc_generated.go

@@ -12,8 +12,9 @@ package v1beta1
 
 // AUTO-GENERATED FUNCTIONS START HERE
 var map_ManagedClusterSelector = map[string]string{
-	"":             "ManagedClusterSelector represents a selector of ManagedClusters",
-	"selectorType": "SelectorType could only be \"LegacyClusterSetLabel\" now, will support more SelectorType later \"LegacyClusterSetLabel\" means to use label \"cluster.open-cluster-management.io/clusterset:<ManagedClusterSet Name>\"\" to select target clusters.",
+	"":              "ManagedClusterSelector represents a selector of ManagedClusters",
+	"selectorType":  "SelectorType could only be \"LegacyClusterSetLabel\" or \"LabelSelector\" \"LegacyClusterSetLabel\" means to use label \"cluster.open-cluster-management.io/clusterset:<ManagedClusterSet Name>\"\" to select target clusters. \"LabelSelector\" means use labelSelector to select target managedClusters",
+	"labelSelector": "LabelSelector define the general labelSelector which clusterset will use to select target managedClusters",
 }
 
 func (ManagedClusterSelector) SwaggerDoc() map[string]string {
@@ -59,8 +60,9 @@ func (ManagedClusterSetStatus) SwaggerDoc() map[string]string {
 }
 
 var map_ManagedClusterSetBinding = map[string]string{
-	"":     "ManagedClusterSetBinding projects a ManagedClusterSet into a certain namespace. User is able to create a ManagedClusterSetBinding in a namespace and bind it to a ManagedClusterSet if they have an RBAC rule to CREATE on the virtual subresource of managedclustersets/bind. Workloads created in the same namespace can only be distributed to ManagedClusters in ManagedClusterSets bound in this namespace by higher level controllers.",
-	"spec": "Spec defines the attributes of ManagedClusterSetBinding.",
+	"":       "ManagedClusterSetBinding projects a ManagedClusterSet into a certain namespace. User is able to create a ManagedClusterSetBinding in a namespace and bind it to a ManagedClusterSet if they have an RBAC rule to CREATE on the virtual subresource of managedclustersets/bind. Workloads created in the same namespace can only be distributed to ManagedClusters in ManagedClusterSets bound in this namespace by higher level controllers.",
+	"spec":   "Spec defines the attributes of ManagedClusterSetBinding.",
+	"status": "Status represents the current status of the ManagedClusterSetBinding",
 }
 
 func (ManagedClusterSetBinding) SwaggerDoc() map[string]string {
@@ -86,6 +88,15 @@ func (ManagedClusterSetBindingSpec) SwaggerDoc() map[string]string {
 	return map_ManagedClusterSetBindingSpec
 }
 
+var map_ManagedClusterSetBindingStatus = map[string]string{
+	"":           "ManagedClusterSetBindingStatus represents the current status of the ManagedClusterSetBinding.",
+	"conditions": "Conditions contains the different condition statuses for this ManagedClusterSetBinding.",
+}
+
+func (ManagedClusterSetBindingStatus) SwaggerDoc() map[string]string {
+	return map_ManagedClusterSetBindingStatus
+}
+
 var map_AddOnScore = map[string]string{
 	"":             "AddOnScore represents the configuration of the addon score source.",
 	"resourceName": "ResourceName defines the resource name of the AddOnPlacementScore. The placement prioritizer selects AddOnPlacementScore CR by this name.",