From 985dcf71cb145b9dadbf4409487908c4fe75e359 Mon Sep 17 00:00:00 2001 From: Yang Le Date: Wed, 12 May 2021 22:10:13 +0800 Subject: [PATCH] deploy placement controller Signed-off-by: Yang Le --- ...cluster-management_clustermanagers.cr.yaml | 1 + ...cluster-manager.clusterserviceversion.yaml | 1 + go.sum | 1 - ...-cluster-management.io_placements.crd.yaml | 274 +++++++++ ...-management.io_placementdecisions.crd.yaml | 76 +++ ...cluster-manager-placement-clusterrole.yaml | 24 + ...-manager-placement-clusterrolebinding.yaml | 12 + .../cluster-manager-placement-deployment.yaml | 69 +++ ...ster-manager-placement-serviceaccount.yaml | 5 + pkg/helpers/queuekey.go | 2 +- .../clustermanager/bindata/bindata.go | 580 ++++++++++++++++++ .../clustermanager_controller.go | 8 + .../clustermanager_controller_test.go | 13 +- .../clustermanager_status_controller.go | 64 +- .../clustermanager_status_controller_test.go | 67 +- test/e2e/common.go | 24 + test/e2e/e2e_suite_test.go | 4 + 17 files changed, 1198 insertions(+), 27 deletions(-) create mode 100644 manifests/cluster-manager/0000_03_clusters.open-cluster-management.io_placements.crd.yaml create mode 100644 manifests/cluster-manager/0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml create mode 100644 manifests/cluster-manager/cluster-manager-placement-clusterrole.yaml create mode 100644 manifests/cluster-manager/cluster-manager-placement-clusterrolebinding.yaml create mode 100644 manifests/cluster-manager/cluster-manager-placement-deployment.yaml create mode 100644 manifests/cluster-manager/cluster-manager-placement-serviceaccount.yaml diff --git a/deploy/cluster-manager/config/samples/operator_open-cluster-management_clustermanagers.cr.yaml b/deploy/cluster-manager/config/samples/operator_open-cluster-management_clustermanagers.cr.yaml index 3b12effab..130f83933 100644 --- a/deploy/cluster-manager/config/samples/operator_open-cluster-management_clustermanagers.cr.yaml +++ b/deploy/cluster-manager/config/samples/operator_open-cluster-management_clustermanagers.cr.yaml @@ -5,3 +5,4 @@ metadata: spec: registrationImagePullSpec: quay.io/open-cluster-management/registration workImagePullSpec: quay.io/open-cluster-management/work + placementImagePullSpec: quay.io/open-cluster-management/placement diff --git a/deploy/cluster-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml b/deploy/cluster-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml index 7fee19833..db75f453c 100644 --- a/deploy/cluster-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml +++ b/deploy/cluster-manager/olm-catalog/cluster-manager/manifests/cluster-manager.clusterserviceversion.yaml @@ -11,6 +11,7 @@ metadata: "name": "cluster-manager" }, "spec": { + "placementImagePullSpec": "quay.io/open-cluster-management/placement", "registrationImagePullSpec": "quay.io/open-cluster-management/registration", "workImagePullSpec": "quay.io/open-cluster-management/work" } diff --git a/go.sum b/go.sum index d456cb085..9242575c6 100644 --- a/go.sum +++ b/go.sum @@ -554,7 +554,6 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= diff --git a/manifests/cluster-manager/0000_03_clusters.open-cluster-management.io_placements.crd.yaml b/manifests/cluster-manager/0000_03_clusters.open-cluster-management.io_placements.crd.yaml new file mode 100644 index 000000000..619a15a89 --- /dev/null +++ b/manifests/cluster-manager/0000_03_clusters.open-cluster-management.io_placements.crd.yaml @@ -0,0 +1,274 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: placements.cluster.open-cluster-management.io +spec: + group: cluster.open-cluster-management.io + names: + kind: Placement + listKind: PlacementList + plural: placements + singular: placement + scope: Namespaced + preserveUnknownFields: false + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: "Placement defines a rule to select a set of ManagedClusters + from the ManagedClusterSets bound to the placement namespace. \n Here is + how the placement policy combines with other selection methods to determine + a matching list of ManagedClusters: 1) Kubernetes clusters are registered + with hub as cluster-scoped ManagedClusters; 2) ManagedClusters are organized + into cluster-scoped ManagedClusterSets; 3) ManagedClusterSets are bound + to workload namespaces; 4) Namespace-scoped Placements specify a slice of + ManagedClusterSets which select a working set of potential ManagedClusters; + 5) Then Placements subselect from that working set using label/claim selection. + \n No ManagedCluster will be selected if no ManagedClusterSet is bound to + the placement namespace. User is able to bind a ManagedClusterSet to a namespace + by creating a ManagedClusterSetBinding in that namespace if they have a + RBAC rule to CREATE on the virtual subresource of `managedclustersets/bind`. + \n A slice of PlacementDecisions with label cluster.open-cluster-management.io/placement={placement + name} will be created to represent the ManagedClusters selected by this + placement. \n If a ManagedCluster is selected and added into the PlacementDecisions, + other components may apply workload on it; once it is removed from the PlacementDecisions, + the workload applied on this ManagedCluster should be evicted accordingly." + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the attributes of Placement. + type: object + properties: + clusterSets: + description: ClusterSets represent the ManagedClusterSets from which + the ManagedClusters are selected. If the slice is empty, ManagedClusters + will be selected from the ManagedClusterSets bound to the placement + namespace, otherwise ManagedClusters will be selected from the intersection + of this slice and the ManagedClusterSets bound to the placement + namespace. + type: array + items: + type: string + numberOfClusters: + description: NumberOfClusters represents the desired number of ManagedClusters + to be selected which meet the placement requirements. 1) If not + specified, all ManagedClusters which meet the placement requirements + (including ClusterSets, and Predicates) will be selected; 2) + Otherwise if the nubmer of ManagedClusters meet the placement requirements + is larger than NumberOfClusters, a random subset with desired + number of ManagedClusters will be selected; 3) If the nubmer of + ManagedClusters meet the placement requirements is equal to NumberOfClusters, all + of them will be selected; 4) If the nubmer of ManagedClusters meet + the placement requirements is less than NumberOfClusters, all + of them will be selected, and the status of condition `PlacementConditionSatisfied` + will be set to false; + type: integer + format: int32 + predicates: + description: Predicates represent a slice of predicates to select + ManagedClusters. The predicates are ORed. + type: array + items: + description: ClusterPredicate represents a predicate to select ManagedClusters. + type: object + properties: + requiredClusterSelector: + description: RequiredClusterSelector represents a selector of + ManagedClusters by label and claim. If specified, 1) Any ManagedCluster, + which does not match the selector, should not be selected + by this ClusterPredicate; 2) If a selected ManagedCluster + (of this ClusterPredicate) ceases to match the selector (e.g. + due to an update) of any ClusterPredicate, it will be eventually + removed from the placement decisions; 3) If a ManagedCluster + (not selected previously) starts to match the selector, it + will either be selected or at least has a chance to be + selected (when NumberOfClusters is specified); + type: object + properties: + claimSelector: + description: ClaimSelector represents a selector of ManagedClusters + by clusterClaims in status + type: object + properties: + matchExpressions: + description: matchExpressions is a list of cluster claim + selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + type: array + items: + type: string + labelSelector: + description: LabelSelector represents a selector of ManagedClusters + by label + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + status: + description: Status represents the current status of the Placement + type: object + properties: + conditions: + description: Conditions contains the different condition statuses + for this Placement. + type: array + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: + \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type + \ // +patchStrategy=merge // +listType=map // +listMapKey=type + \ Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + type: object + required: + - lastTransitionTime + - message + - reason + - status + - type + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + type: string + format: date-time + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + type: string + maxLength: 32768 + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + type: integer + format: int64 + minimum: 0 + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + type: string + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: + description: status of the condition, one of True, False, Unknown. + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + numberOfSelectedClusters: + description: NumberOfSelectedClusters represents the number of selected + ManagedClusters + type: integer + format: int32 + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/manifests/cluster-manager/0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml b/manifests/cluster-manager/0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml new file mode 100644 index 000000000..fc75289ea --- /dev/null +++ b/manifests/cluster-manager/0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml @@ -0,0 +1,76 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: placementdecisions.cluster.open-cluster-management.io +spec: + group: cluster.open-cluster-management.io + names: + kind: PlacementDecision + listKind: PlacementDecisionList + plural: placementdecisions + singular: placementdecision + scope: Namespaced + preserveUnknownFields: false + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: "PlacementDecision indicates a decision from a placement PlacementDecision + should has a label cluster.open-cluster-management.io/placement={placement + name} to reference a certain placement. \n If a placement has spec.numberOfClusters + specified, the total number of decisions contained in status.decisions of + PlacementDecisions should always be NumberOfClusters; otherwise, the total + number of decisions should be the number of ManagedClusters which match + the placement requirements. \n Some of the decisions might be empty when + there are no enough ManagedClusters meet the placement requirements." + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: Status represents the current status of the PlacementDecision + type: object + required: + - decisions + properties: + decisions: + description: Decisions is a slice of decisions according to a placement + The number of decisions should not be larger than 100 + type: array + items: + description: ClusterDecision represents a decision from a placement + An empty ClusterDecision indicates it is not scheduled yet. + type: object + required: + - clusterName + - reason + properties: + clusterName: + description: ClusterName is the name of the ManagedCluster. + If it is not empty, its value should be unique cross all placement + decisions for the Placement. + type: string + reason: + description: Reason represents the reason why the ManagedCluster + is selected. + type: string + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/manifests/cluster-manager/cluster-manager-placement-clusterrole.yaml b/manifests/cluster-manager/cluster-manager-placement-clusterrole.yaml new file mode 100644 index 000000000..71aecddc3 --- /dev/null +++ b/manifests/cluster-manager/cluster-manager-placement-clusterrole.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: open-cluster-management:{{ .ClusterManagerName }}-placement:controller +rules: +# Allow controller to get/list/watch/create/delete configmaps +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update"] +# Allow controller to create/patch/update events +- apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch", "update"] +# Allow controller to view managedclusters/managedclustersets/managedclustersetbindings +- apiGroups: ["cluster.open-cluster-management.io"] + resources: ["managedclusters", "managedclustersets", "managedclustersetbindings"] + verbs: ["get", "list", "watch"] +# Allow controller to manage placements/placementdecisions +- apiGroups: ["cluster.open-cluster-management.io"] + resources: ["placements", "placementdecisions"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: ["cluster.open-cluster-management.io"] + resources: ["placements/status", "placementdecisions/status"] + verbs: ["update", "patch"] diff --git a/manifests/cluster-manager/cluster-manager-placement-clusterrolebinding.yaml b/manifests/cluster-manager/cluster-manager-placement-clusterrolebinding.yaml new file mode 100644 index 000000000..e9d337853 --- /dev/null +++ b/manifests/cluster-manager/cluster-manager-placement-clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: open-cluster-management:{{ .ClusterManagerName }}-placement:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: open-cluster-management:{{ .ClusterManagerName }}-placement:controller +subjects: +- kind: ServiceAccount + namespace: open-cluster-management-hub + name: {{ .ClusterManagerName }}-placement-controller-sa diff --git a/manifests/cluster-manager/cluster-manager-placement-deployment.yaml b/manifests/cluster-manager/cluster-manager-placement-deployment.yaml new file mode 100644 index 000000000..28412edac --- /dev/null +++ b/manifests/cluster-manager/cluster-manager-placement-deployment.yaml @@ -0,0 +1,69 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .ClusterManagerName }}-placement-controller + namespace: open-cluster-management-hub + labels: + app: clustermanager-controller +spec: + replicas: 3 + selector: + matchLabels: + app: clustermanager-placement-controller + template: + metadata: + labels: + app: clustermanager-placement-controller + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 70 + podAffinityTerm: + topologyKey: failure-domain.beta.kubernetes.io/zone + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - clustermanager-placement-controller + - weight: 30 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - clustermanager-placement-controller + serviceAccountName: {{ .ClusterManagerName }}-placement-controller-sa + containers: + - name: placement-controller + image: {{ .PlacementImage }} + args: + - "/placement" + - "controller" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsNonRoot: true + livenessProbe: + httpGet: + path: /healthz + scheme: HTTPS + port: 8443 + initialDelaySeconds: 2 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + scheme: HTTPS + port: 8443 + initialDelaySeconds: 2 + resources: + requests: + cpu: 100m + memory: 128Mi diff --git a/manifests/cluster-manager/cluster-manager-placement-serviceaccount.yaml b/manifests/cluster-manager/cluster-manager-placement-serviceaccount.yaml new file mode 100644 index 000000000..90be217fe --- /dev/null +++ b/manifests/cluster-manager/cluster-manager-placement-serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .ClusterManagerName }}-placement-controller-sa + namespace: open-cluster-management-hub diff --git a/pkg/helpers/queuekey.go b/pkg/helpers/queuekey.go index 51ce03f76..ad1c42d55 100644 --- a/pkg/helpers/queuekey.go +++ b/pkg/helpers/queuekey.go @@ -90,7 +90,7 @@ func ClusterManagerDeploymentQueueKeyFunc(clusterManagerLister operatorlister.Cl if namespace != ClusterManagerNamespace { return "" } - if strings.HasSuffix(name, "registration-controller") || strings.HasSuffix(name, "work-controller") { + if strings.HasSuffix(name, "registration-controller") || strings.HasSuffix(name, "work-controller") || strings.HasSuffix(name, "placement-controller") { interestedObjectFound = true } if !interestedObjectFound { diff --git a/pkg/operators/clustermanager/bindata/bindata.go b/pkg/operators/clustermanager/bindata/bindata.go index bd6e0c7ce..2e471b360 100644 --- a/pkg/operators/clustermanager/bindata/bindata.go +++ b/pkg/operators/clustermanager/bindata/bindata.go @@ -6,7 +6,13 @@ // manifests/cluster-manager/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml // manifests/cluster-manager/0000_01_addon.open-cluster-management.io_managedclusteraddons.crd.yaml // manifests/cluster-manager/0000_01_clusters.open-cluster-management.io_managedclustersetbindings.crd.yaml +// manifests/cluster-manager/0000_03_clusters.open-cluster-management.io_placements.crd.yaml +// manifests/cluster-manager/0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml // manifests/cluster-manager/cluster-manager-namespace.yaml +// manifests/cluster-manager/cluster-manager-placement-clusterrole.yaml +// manifests/cluster-manager/cluster-manager-placement-clusterrolebinding.yaml +// manifests/cluster-manager/cluster-manager-placement-deployment.yaml +// manifests/cluster-manager/cluster-manager-placement-serviceaccount.yaml // manifests/cluster-manager/cluster-manager-registration-clusterrole.yaml // manifests/cluster-manager/cluster-manager-registration-clusterrolebinding.yaml // manifests/cluster-manager/cluster-manager-registration-deployment.yaml @@ -1237,6 +1243,390 @@ func manifestsClusterManager0000_01_clustersOpenClusterManagementIo_managedclust return a, nil } +var _manifestsClusterManager0000_03_clustersOpenClusterManagementIo_placementsCrdYaml = []byte(`apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: placements.cluster.open-cluster-management.io +spec: + group: cluster.open-cluster-management.io + names: + kind: Placement + listKind: PlacementList + plural: placements + singular: placement + scope: Namespaced + preserveUnknownFields: false + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: "Placement defines a rule to select a set of ManagedClusters + from the ManagedClusterSets bound to the placement namespace. \n Here is + how the placement policy combines with other selection methods to determine + a matching list of ManagedClusters: 1) Kubernetes clusters are registered + with hub as cluster-scoped ManagedClusters; 2) ManagedClusters are organized + into cluster-scoped ManagedClusterSets; 3) ManagedClusterSets are bound + to workload namespaces; 4) Namespace-scoped Placements specify a slice of + ManagedClusterSets which select a working set of potential ManagedClusters; + 5) Then Placements subselect from that working set using label/claim selection. + \n No ManagedCluster will be selected if no ManagedClusterSet is bound to + the placement namespace. User is able to bind a ManagedClusterSet to a namespace + by creating a ManagedClusterSetBinding in that namespace if they have a + RBAC rule to CREATE on the virtual subresource of ` + "`" + `managedclustersets/bind` + "`" + `. + \n A slice of PlacementDecisions with label cluster.open-cluster-management.io/placement={placement + name} will be created to represent the ManagedClusters selected by this + placement. \n If a ManagedCluster is selected and added into the PlacementDecisions, + other components may apply workload on it; once it is removed from the PlacementDecisions, + the workload applied on this ManagedCluster should be evicted accordingly." + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the attributes of Placement. + type: object + properties: + clusterSets: + description: ClusterSets represent the ManagedClusterSets from which + the ManagedClusters are selected. If the slice is empty, ManagedClusters + will be selected from the ManagedClusterSets bound to the placement + namespace, otherwise ManagedClusters will be selected from the intersection + of this slice and the ManagedClusterSets bound to the placement + namespace. + type: array + items: + type: string + numberOfClusters: + description: NumberOfClusters represents the desired number of ManagedClusters + to be selected which meet the placement requirements. 1) If not + specified, all ManagedClusters which meet the placement requirements + (including ClusterSets, and Predicates) will be selected; 2) + Otherwise if the nubmer of ManagedClusters meet the placement requirements + is larger than NumberOfClusters, a random subset with desired + number of ManagedClusters will be selected; 3) If the nubmer of + ManagedClusters meet the placement requirements is equal to NumberOfClusters, all + of them will be selected; 4) If the nubmer of ManagedClusters meet + the placement requirements is less than NumberOfClusters, all + of them will be selected, and the status of condition ` + "`" + `PlacementConditionSatisfied` + "`" + ` + will be set to false; + type: integer + format: int32 + predicates: + description: Predicates represent a slice of predicates to select + ManagedClusters. The predicates are ORed. + type: array + items: + description: ClusterPredicate represents a predicate to select ManagedClusters. + type: object + properties: + requiredClusterSelector: + description: RequiredClusterSelector represents a selector of + ManagedClusters by label and claim. If specified, 1) Any ManagedCluster, + which does not match the selector, should not be selected + by this ClusterPredicate; 2) If a selected ManagedCluster + (of this ClusterPredicate) ceases to match the selector (e.g. + due to an update) of any ClusterPredicate, it will be eventually + removed from the placement decisions; 3) If a ManagedCluster + (not selected previously) starts to match the selector, it + will either be selected or at least has a chance to be + selected (when NumberOfClusters is specified); + type: object + properties: + claimSelector: + description: ClaimSelector represents a selector of ManagedClusters + by clusterClaims in status + type: object + properties: + matchExpressions: + description: matchExpressions is a list of cluster claim + selector requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + type: array + items: + type: string + labelSelector: + description: LabelSelector represents a selector of ManagedClusters + by label + type: object + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + type: array + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + type: object + required: + - key + - operator + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + type: array + items: + type: string + matchLabels: + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + additionalProperties: + type: string + status: + description: Status represents the current status of the Placement + type: object + properties: + conditions: + description: Conditions contains the different condition statuses + for this Placement. + type: array + items: + description: "Condition contains details for one aspect of the current + state of this API Resource. --- This struct is intended for direct + use as an array at the field path .status.conditions. For example, + type FooStatus struct{ // Represents the observations of a + foo's current state. // Known .status.conditions.type are: + \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type + \ // +patchStrategy=merge // +listType=map // +listMapKey=type + \ Conditions []metav1.Condition ` + "`" + `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + "`" + ` + \n // other fields }" + type: object + required: + - lastTransitionTime + - message + - reason + - status + - type + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another. This should be when + the underlying condition changed. If that is not known, then + using the time when the API field changed is acceptable. + type: string + format: date-time + message: + description: message is a human readable message indicating + details about the transition. This may be an empty string. + type: string + maxLength: 32768 + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, if .metadata.generation + is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to the current + state of the instance. + type: integer + format: int64 + minimum: 0 + reason: + description: reason contains a programmatic identifier indicating + the reason for the condition's last transition. Producers + of specific condition types may define expected values and + meanings for this field, and whether the values are considered + a guaranteed API. The value should be a CamelCase string. + This field may not be empty. + type: string + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: + description: status of the condition, one of True, False, Unknown. + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + numberOfSelectedClusters: + description: NumberOfSelectedClusters represents the number of selected + ManagedClusters + type: integer + format: int32 + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +`) + +func manifestsClusterManager0000_03_clustersOpenClusterManagementIo_placementsCrdYamlBytes() ([]byte, error) { + return _manifestsClusterManager0000_03_clustersOpenClusterManagementIo_placementsCrdYaml, nil +} + +func manifestsClusterManager0000_03_clustersOpenClusterManagementIo_placementsCrdYaml() (*asset, error) { + bytes, err := manifestsClusterManager0000_03_clustersOpenClusterManagementIo_placementsCrdYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/cluster-manager/0000_03_clusters.open-cluster-management.io_placements.crd.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsClusterManager0000_04_clustersOpenClusterManagementIo_placementdecisionsCrdYaml = []byte(`apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: placementdecisions.cluster.open-cluster-management.io +spec: + group: cluster.open-cluster-management.io + names: + kind: PlacementDecision + listKind: PlacementDecisionList + plural: placementdecisions + singular: placementdecision + scope: Namespaced + preserveUnknownFields: false + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: "PlacementDecision indicates a decision from a placement PlacementDecision + should has a label cluster.open-cluster-management.io/placement={placement + name} to reference a certain placement. \n If a placement has spec.numberOfClusters + specified, the total number of decisions contained in status.decisions of + PlacementDecisions should always be NumberOfClusters; otherwise, the total + number of decisions should be the number of ManagedClusters which match + the placement requirements. \n Some of the decisions might be empty when + there are no enough ManagedClusters meet the placement requirements." + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + description: Status represents the current status of the PlacementDecision + type: object + required: + - decisions + properties: + decisions: + description: Decisions is a slice of decisions according to a placement + The number of decisions should not be larger than 100 + type: array + items: + description: ClusterDecision represents a decision from a placement + An empty ClusterDecision indicates it is not scheduled yet. + type: object + required: + - clusterName + - reason + properties: + clusterName: + description: ClusterName is the name of the ManagedCluster. + If it is not empty, its value should be unique cross all placement + decisions for the Placement. + type: string + reason: + description: Reason represents the reason why the ManagedCluster + is selected. + type: string + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +`) + +func manifestsClusterManager0000_04_clustersOpenClusterManagementIo_placementdecisionsCrdYamlBytes() ([]byte, error) { + return _manifestsClusterManager0000_04_clustersOpenClusterManagementIo_placementdecisionsCrdYaml, nil +} + +func manifestsClusterManager0000_04_clustersOpenClusterManagementIo_placementdecisionsCrdYaml() (*asset, error) { + bytes, err := manifestsClusterManager0000_04_clustersOpenClusterManagementIo_placementdecisionsCrdYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/cluster-manager/0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _manifestsClusterManagerClusterManagerNamespaceYaml = []byte(`apiVersion: v1 kind: Namespace metadata: @@ -1258,6 +1648,184 @@ func manifestsClusterManagerClusterManagerNamespaceYaml() (*asset, error) { return a, nil } +var _manifestsClusterManagerClusterManagerPlacementClusterroleYaml = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: open-cluster-management:{{ .ClusterManagerName }}-placement:controller +rules: +# Allow controller to get/list/watch/create/delete configmaps +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update"] +# Allow controller to create/patch/update events +- apiGroups: ["", "events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch", "update"] +# Allow controller to view managedclusters/managedclustersets/managedclustersetbindings +- apiGroups: ["cluster.open-cluster-management.io"] + resources: ["managedclusters", "managedclustersets", "managedclustersetbindings"] + verbs: ["get", "list", "watch"] +# Allow controller to manage placements/placementdecisions +- apiGroups: ["cluster.open-cluster-management.io"] + resources: ["placements", "placementdecisions"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: ["cluster.open-cluster-management.io"] + resources: ["placements/status", "placementdecisions/status"] + verbs: ["update", "patch"] +`) + +func manifestsClusterManagerClusterManagerPlacementClusterroleYamlBytes() ([]byte, error) { + return _manifestsClusterManagerClusterManagerPlacementClusterroleYaml, nil +} + +func manifestsClusterManagerClusterManagerPlacementClusterroleYaml() (*asset, error) { + bytes, err := manifestsClusterManagerClusterManagerPlacementClusterroleYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/cluster-manager/cluster-manager-placement-clusterrole.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsClusterManagerClusterManagerPlacementClusterrolebindingYaml = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: open-cluster-management:{{ .ClusterManagerName }}-placement:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: open-cluster-management:{{ .ClusterManagerName }}-placement:controller +subjects: +- kind: ServiceAccount + namespace: open-cluster-management-hub + name: {{ .ClusterManagerName }}-placement-controller-sa +`) + +func manifestsClusterManagerClusterManagerPlacementClusterrolebindingYamlBytes() ([]byte, error) { + return _manifestsClusterManagerClusterManagerPlacementClusterrolebindingYaml, nil +} + +func manifestsClusterManagerClusterManagerPlacementClusterrolebindingYaml() (*asset, error) { + bytes, err := manifestsClusterManagerClusterManagerPlacementClusterrolebindingYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/cluster-manager/cluster-manager-placement-clusterrolebinding.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsClusterManagerClusterManagerPlacementDeploymentYaml = []byte(`kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .ClusterManagerName }}-placement-controller + namespace: open-cluster-management-hub + labels: + app: clustermanager-controller +spec: + replicas: 3 + selector: + matchLabels: + app: clustermanager-placement-controller + template: + metadata: + labels: + app: clustermanager-placement-controller + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 70 + podAffinityTerm: + topologyKey: failure-domain.beta.kubernetes.io/zone + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - clustermanager-placement-controller + - weight: 30 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - clustermanager-placement-controller + serviceAccountName: {{ .ClusterManagerName }}-placement-controller-sa + containers: + - name: placement-controller + image: {{ .PlacementImage }} + args: + - "/placement" + - "controller" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsNonRoot: true + livenessProbe: + httpGet: + path: /healthz + scheme: HTTPS + port: 8443 + initialDelaySeconds: 2 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + scheme: HTTPS + port: 8443 + initialDelaySeconds: 2 + resources: + requests: + cpu: 100m + memory: 128Mi +`) + +func manifestsClusterManagerClusterManagerPlacementDeploymentYamlBytes() ([]byte, error) { + return _manifestsClusterManagerClusterManagerPlacementDeploymentYaml, nil +} + +func manifestsClusterManagerClusterManagerPlacementDeploymentYaml() (*asset, error) { + bytes, err := manifestsClusterManagerClusterManagerPlacementDeploymentYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/cluster-manager/cluster-manager-placement-deployment.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _manifestsClusterManagerClusterManagerPlacementServiceaccountYaml = []byte(`apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .ClusterManagerName }}-placement-controller-sa + namespace: open-cluster-management-hub +`) + +func manifestsClusterManagerClusterManagerPlacementServiceaccountYamlBytes() ([]byte, error) { + return _manifestsClusterManagerClusterManagerPlacementServiceaccountYaml, nil +} + +func manifestsClusterManagerClusterManagerPlacementServiceaccountYaml() (*asset, error) { + bytes, err := manifestsClusterManagerClusterManagerPlacementServiceaccountYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/cluster-manager/cluster-manager-placement-serviceaccount.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _manifestsClusterManagerClusterManagerRegistrationClusterroleYaml = []byte(`apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -2186,7 +2754,13 @@ var _bindata = map[string]func() (*asset, error){ "manifests/cluster-manager/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml": manifestsClusterManager0000_00_workOpenClusterManagementIo_manifestworksCrdYaml, "manifests/cluster-manager/0000_01_addon.open-cluster-management.io_managedclusteraddons.crd.yaml": manifestsClusterManager0000_01_addonOpenClusterManagementIo_managedclusteraddonsCrdYaml, "manifests/cluster-manager/0000_01_clusters.open-cluster-management.io_managedclustersetbindings.crd.yaml": manifestsClusterManager0000_01_clustersOpenClusterManagementIo_managedclustersetbindingsCrdYaml, + "manifests/cluster-manager/0000_03_clusters.open-cluster-management.io_placements.crd.yaml": manifestsClusterManager0000_03_clustersOpenClusterManagementIo_placementsCrdYaml, + "manifests/cluster-manager/0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml": manifestsClusterManager0000_04_clustersOpenClusterManagementIo_placementdecisionsCrdYaml, "manifests/cluster-manager/cluster-manager-namespace.yaml": manifestsClusterManagerClusterManagerNamespaceYaml, + "manifests/cluster-manager/cluster-manager-placement-clusterrole.yaml": manifestsClusterManagerClusterManagerPlacementClusterroleYaml, + "manifests/cluster-manager/cluster-manager-placement-clusterrolebinding.yaml": manifestsClusterManagerClusterManagerPlacementClusterrolebindingYaml, + "manifests/cluster-manager/cluster-manager-placement-deployment.yaml": manifestsClusterManagerClusterManagerPlacementDeploymentYaml, + "manifests/cluster-manager/cluster-manager-placement-serviceaccount.yaml": manifestsClusterManagerClusterManagerPlacementServiceaccountYaml, "manifests/cluster-manager/cluster-manager-registration-clusterrole.yaml": manifestsClusterManagerClusterManagerRegistrationClusterroleYaml, "manifests/cluster-manager/cluster-manager-registration-clusterrolebinding.yaml": manifestsClusterManagerClusterManagerRegistrationClusterrolebindingYaml, "manifests/cluster-manager/cluster-manager-registration-deployment.yaml": manifestsClusterManagerClusterManagerRegistrationDeploymentYaml, @@ -2258,7 +2832,13 @@ var _bintree = &bintree{nil, map[string]*bintree{ "0000_00_work.open-cluster-management.io_manifestworks.crd.yaml": {manifestsClusterManager0000_00_workOpenClusterManagementIo_manifestworksCrdYaml, map[string]*bintree{}}, "0000_01_addon.open-cluster-management.io_managedclusteraddons.crd.yaml": {manifestsClusterManager0000_01_addonOpenClusterManagementIo_managedclusteraddonsCrdYaml, map[string]*bintree{}}, "0000_01_clusters.open-cluster-management.io_managedclustersetbindings.crd.yaml": {manifestsClusterManager0000_01_clustersOpenClusterManagementIo_managedclustersetbindingsCrdYaml, map[string]*bintree{}}, + "0000_03_clusters.open-cluster-management.io_placements.crd.yaml": {manifestsClusterManager0000_03_clustersOpenClusterManagementIo_placementsCrdYaml, map[string]*bintree{}}, + "0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml": {manifestsClusterManager0000_04_clustersOpenClusterManagementIo_placementdecisionsCrdYaml, map[string]*bintree{}}, "cluster-manager-namespace.yaml": {manifestsClusterManagerClusterManagerNamespaceYaml, map[string]*bintree{}}, + "cluster-manager-placement-clusterrole.yaml": {manifestsClusterManagerClusterManagerPlacementClusterroleYaml, map[string]*bintree{}}, + "cluster-manager-placement-clusterrolebinding.yaml": {manifestsClusterManagerClusterManagerPlacementClusterrolebindingYaml, map[string]*bintree{}}, + "cluster-manager-placement-deployment.yaml": {manifestsClusterManagerClusterManagerPlacementDeploymentYaml, map[string]*bintree{}}, + "cluster-manager-placement-serviceaccount.yaml": {manifestsClusterManagerClusterManagerPlacementServiceaccountYaml, map[string]*bintree{}}, "cluster-manager-registration-clusterrole.yaml": {manifestsClusterManagerClusterManagerRegistrationClusterroleYaml, map[string]*bintree{}}, "cluster-manager-registration-clusterrolebinding.yaml": {manifestsClusterManagerClusterManagerRegistrationClusterrolebindingYaml, map[string]*bintree{}}, "cluster-manager-registration-deployment.yaml": {manifestsClusterManagerClusterManagerRegistrationDeploymentYaml, map[string]*bintree{}}, diff --git a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go index e82001442..9f1afc593 100644 --- a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go +++ b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller.go @@ -44,6 +44,8 @@ var ( "manifests/cluster-manager/0000_00_work.open-cluster-management.io_manifestworks.crd.yaml", "manifests/cluster-manager/0000_01_addon.open-cluster-management.io_managedclusteraddons.crd.yaml", "manifests/cluster-manager/0000_01_clusters.open-cluster-management.io_managedclustersetbindings.crd.yaml", + "manifests/cluster-manager/0000_03_clusters.open-cluster-management.io_placements.crd.yaml", + "manifests/cluster-manager/0000_04_clusters.open-cluster-management.io_placementdecisions.crd.yaml", "manifests/cluster-manager/cluster-manager-registration-clusterrole.yaml", "manifests/cluster-manager/cluster-manager-registration-clusterrolebinding.yaml", "manifests/cluster-manager/cluster-manager-namespace.yaml", @@ -62,12 +64,16 @@ var ( "manifests/cluster-manager/cluster-manager-work-webhook-serviceaccount.yaml", "manifests/cluster-manager/cluster-manager-work-webhook-apiservice.yaml", "manifests/cluster-manager/cluster-manager-work-webhook-validatingconfiguration.yaml", + "manifests/cluster-manager/cluster-manager-placement-clusterrole.yaml", + "manifests/cluster-manager/cluster-manager-placement-clusterrolebinding.yaml", + "manifests/cluster-manager/cluster-manager-placement-serviceaccount.yaml", } deploymentFiles = []string{ "manifests/cluster-manager/cluster-manager-registration-deployment.yaml", "manifests/cluster-manager/cluster-manager-registration-webhook-deployment.yaml", "manifests/cluster-manager/cluster-manager-work-webhook-deployment.yaml", + "manifests/cluster-manager/cluster-manager-placement-deployment.yaml", } ) @@ -138,6 +144,7 @@ type hubConfig struct { RegistrationAPIServiceCABundle string WorkImage string WorkAPIServiceCABundle string + PlacementImage string } func (n *clusterManagerController) sync(ctx context.Context, controllerContext factory.SyncContext) error { @@ -158,6 +165,7 @@ func (n *clusterManagerController) sync(ctx context.Context, controllerContext f ClusterManagerName: clusterManager.Name, RegistrationImage: clusterManager.Spec.RegistrationImagePullSpec, WorkImage: clusterManager.Spec.WorkImagePullSpec, + PlacementImage: clusterManager.Spec.PlacementImagePullSpec, } // Update finalizer at first diff --git a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go index d510ec510..1461acf72 100644 --- a/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go +++ b/pkg/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_controller_test.go @@ -99,7 +99,10 @@ func ensureObject(t *testing.T, object runtime.Object, hubCore *operatorapiv1.Cl testinghelper.AssertEqualNameNamespace(t, access.GetName(), "", helpers.ClusterManagerNamespace, "") case *appsv1.Deployment: if strings.Contains(o.Name, "registration") && hubCore.Spec.RegistrationImagePullSpec != o.Spec.Template.Spec.Containers[0].Image { - t.Errorf("Image does not match to the expected.") + t.Errorf("Registration image does not match to the expected.") + } + if strings.Contains(o.Name, "placement") && hubCore.Spec.PlacementImagePullSpec != o.Spec.Template.Spec.Containers[0].Image { + t.Errorf("Placement image does not match to the expected.") } } } @@ -125,7 +128,7 @@ func TestSyncDeploy(t *testing.T) { } // Check if resources are created as expected - testinghelper.AssertEqualNumber(t, len(createKubeObjects), 19) + testinghelper.AssertEqualNumber(t, len(createKubeObjects), 23) for _, object := range createKubeObjects { ensureObject(t, object, clusterManager) } @@ -139,7 +142,7 @@ func TestSyncDeploy(t *testing.T) { } } // Check if resources are created as expected - testinghelper.AssertEqualNumber(t, len(createCRDObjects), 6) + testinghelper.AssertEqualNumber(t, len(createCRDObjects), 8) createAPIServiceObjects := []runtime.Object{} apiServiceActions := controller.apiRegistrationClient.Actions() @@ -181,7 +184,7 @@ func TestSyncDelete(t *testing.T) { deleteKubeActions = append(deleteKubeActions, deleteKubeAction) } } - testinghelper.AssertEqualNumber(t, len(deleteKubeActions), 16) + testinghelper.AssertEqualNumber(t, len(deleteKubeActions), 19) deleteCRDActions := []clienttesting.DeleteActionImpl{} crdActions := controller.apiExtensionClient.Actions() @@ -192,7 +195,7 @@ func TestSyncDelete(t *testing.T) { } } // Check if resources are created as expected - testinghelper.AssertEqualNumber(t, len(deleteCRDActions), 8) + testinghelper.AssertEqualNumber(t, len(deleteCRDActions), 10) deleteAPIServiceActions := []clienttesting.DeleteActionImpl{} apiServiceActions := controller.apiRegistrationClient.Actions() diff --git a/pkg/operators/clustermanager/controllers/statuscontroller/clustermanager_status_controller.go b/pkg/operators/clustermanager/controllers/statuscontroller/clustermanager_status_controller.go index accda0b98..eef76650f 100644 --- a/pkg/operators/clustermanager/controllers/statuscontroller/clustermanager_status_controller.go +++ b/pkg/operators/clustermanager/controllers/statuscontroller/clustermanager_status_controller.go @@ -19,9 +19,11 @@ import ( "github.com/openshift/library-go/pkg/controller/factory" "github.com/openshift/library-go/pkg/operator/events" + operatorhelpers "github.com/openshift/library-go/pkg/operator/v1helpers" ) const registrationDegraded = "HubRegistrationDegraded" +const placementDegraded = "HubPlacementDegraded" type clusterManagerStatusController struct { deploymentLister appslister.DeploymentLister @@ -69,11 +71,25 @@ func (s *clusterManagerStatusController) sync(ctx context.Context, controllerCon return err } + errs := []error{} + if err := s.updateStatusOfRegistration(ctx, clusterManager.Name); err != nil { + errs = append(errs, err) + } + + if err := s.updateStatusOfPlacement(ctx, clusterManager.Name); err != nil { + errs = append(errs, err) + } + + return operatorhelpers.NewMultiLineAggregate(errs) +} + +// updateStatusOfRegistration checks registration deployment status and updates condition of clustermanager +func (s *clusterManagerStatusController) updateStatusOfRegistration(ctx context.Context, clusterManagerName string) error { // Check registration deployment status - registrationDeploymentName := fmt.Sprintf("%s-registration-controller", clusterManager.Name) + registrationDeploymentName := fmt.Sprintf("%s-registration-controller", clusterManagerName) registrationDeployment, err := s.deploymentLister.Deployments(helpers.ClusterManagerNamespace).Get(registrationDeploymentName) if err != nil { - _, _, err := helpers.UpdateClusterManagerStatus(ctx, s.clusterManagerClient, clusterManager.Name, + _, _, err := helpers.UpdateClusterManagerStatus(ctx, s.clusterManagerClient, clusterManagerName, helpers.UpdateClusterManagerConditionFn(metav1.Condition{ Type: registrationDegraded, Status: metav1.ConditionTrue, @@ -85,7 +101,7 @@ func (s *clusterManagerStatusController) sync(ctx context.Context, controllerCon } if unavailablePod := helpers.NumOfUnavailablePod(registrationDeployment); unavailablePod > 0 { - _, _, err := helpers.UpdateClusterManagerStatus(ctx, s.clusterManagerClient, clusterManager.Name, + _, _, err := helpers.UpdateClusterManagerStatus(ctx, s.clusterManagerClient, clusterManagerName, helpers.UpdateClusterManagerConditionFn(metav1.Condition{ Type: registrationDegraded, Status: metav1.ConditionTrue, @@ -96,7 +112,7 @@ func (s *clusterManagerStatusController) sync(ctx context.Context, controllerCon return err } - _, _, err = helpers.UpdateClusterManagerStatus(ctx, s.clusterManagerClient, clusterManager.Name, + _, _, err = helpers.UpdateClusterManagerStatus(ctx, s.clusterManagerClient, clusterManagerName, helpers.UpdateClusterManagerConditionFn(metav1.Condition{ Type: registrationDegraded, Status: metav1.ConditionFalse, @@ -106,3 +122,43 @@ func (s *clusterManagerStatusController) sync(ctx context.Context, controllerCon ) return err } + +// updateStatusOfRegistration checks placement deployment status and updates condition of clustermanager +func (s *clusterManagerStatusController) updateStatusOfPlacement(ctx context.Context, clusterManagerName string) error { + // Check registration deployment status + placementDeploymentName := fmt.Sprintf("%s-placement-controller", clusterManagerName) + placementDeployment, err := s.deploymentLister.Deployments(helpers.ClusterManagerNamespace).Get(placementDeploymentName) + if err != nil { + _, _, err := helpers.UpdateClusterManagerStatus(ctx, s.clusterManagerClient, clusterManagerName, + helpers.UpdateClusterManagerConditionFn(metav1.Condition{ + Type: placementDegraded, + Status: metav1.ConditionTrue, + Reason: "GetPlacementDeploymentFailed", + Message: fmt.Sprintf("Failed to get placement deployment %q %q: %v", helpers.ClusterManagerNamespace, placementDeploymentName, err), + }), + ) + return err + } + + if unavailablePod := helpers.NumOfUnavailablePod(placementDeployment); unavailablePod > 0 { + _, _, err := helpers.UpdateClusterManagerStatus(ctx, s.clusterManagerClient, clusterManagerName, + helpers.UpdateClusterManagerConditionFn(metav1.Condition{ + Type: placementDegraded, + Status: metav1.ConditionTrue, + Reason: "UnavailablePlacementPod", + Message: fmt.Sprintf("%v of requested instances are unavailable of placement deployment %q %q", unavailablePod, helpers.ClusterManagerNamespace, placementDeploymentName), + }), + ) + return err + } + + _, _, err = helpers.UpdateClusterManagerStatus(ctx, s.clusterManagerClient, clusterManagerName, + helpers.UpdateClusterManagerConditionFn(metav1.Condition{ + Type: placementDegraded, + Status: metav1.ConditionFalse, + Reason: "PlacementFunctional", + Message: "Placement is scheduling placement decisions", + }), + ) + return err +} diff --git a/pkg/operators/clustermanager/controllers/statuscontroller/clustermanager_status_controller_test.go b/pkg/operators/clustermanager/controllers/statuscontroller/clustermanager_status_controller_test.go index 7201c2ef2..716f3d520 100644 --- a/pkg/operators/clustermanager/controllers/statuscontroller/clustermanager_status_controller_test.go +++ b/pkg/operators/clustermanager/controllers/statuscontroller/clustermanager_status_controller_test.go @@ -32,7 +32,7 @@ func newClusterManager() *operatorapiv1.ClusterManager { } } -func newDeployment(desiredReplica, availableReplica int32) *appsv1.Deployment { +func newRegistrationDeployment(desiredReplica, availableReplica int32) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-registration-controller", testClusterManagerName), @@ -47,6 +47,21 @@ func newDeployment(desiredReplica, availableReplica int32) *appsv1.Deployment { } } +func newPlacementDeployment(desiredReplica, availableReplica int32) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-placement-controller", testClusterManagerName), + Namespace: "open-cluster-management-hub", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &desiredReplica, + }, + Status: appsv1.DeploymentStatus{ + AvailableReplicas: availableReplica, + }, + } +} + func TestSyncStatus(t *testing.T) { cases := []struct { name string @@ -74,42 +89,62 @@ func TestSyncStatus(t *testing.T) { }, }, { - name: "failed to get registration deployment", + name: "no registration deployment and unavailable placement pods", queueKey: testClusterManagerName, clusterManagers: []runtime.Object{newClusterManager()}, - deployments: []runtime.Object{}, + deployments: []runtime.Object{ + newPlacementDeployment(3, 0), + }, validateActions: func(t *testing.T, actions []clienttesting.Action) { - testinghelper.AssertEqualNumber(t, len(actions), 2) + testinghelper.AssertEqualNumber(t, len(actions), 4) testinghelper.AssertGet(t, actions[0], "operator.open-cluster-management.io", "v1", "clustermanagers") testinghelper.AssertAction(t, actions[1], "update") - expectedCondition := testinghelper.NamedCondition(registrationDegraded, "GetRegistrationDeploymentFailed", metav1.ConditionTrue) - testinghelper.AssertOnlyConditions(t, actions[1].(clienttesting.UpdateActionImpl).Object, expectedCondition) + expectedCondition1 := testinghelper.NamedCondition(registrationDegraded, "GetRegistrationDeploymentFailed", metav1.ConditionTrue) + testinghelper.AssertOnlyConditions(t, actions[1].(clienttesting.UpdateActionImpl).Object, expectedCondition1) + + testinghelper.AssertGet(t, actions[2], "operator.open-cluster-management.io", "v1", "clustermanagers") + testinghelper.AssertAction(t, actions[3], "update") + expectedCondition2 := testinghelper.NamedCondition(placementDegraded, "UnavailablePlacementPod", metav1.ConditionTrue) + testinghelper.AssertOnlyConditions(t, actions[3].(clienttesting.UpdateActionImpl).Object, expectedCondition1, expectedCondition2) }, }, { - name: "unavailable registration pods", + name: "unavailable registration pods and placement functional", queueKey: testClusterManagerName, clusterManagers: []runtime.Object{newClusterManager()}, - deployments: []runtime.Object{newDeployment(3, 0)}, + deployments: []runtime.Object{ + newRegistrationDeployment(3, 0), + newPlacementDeployment(3, 3), + }, validateActions: func(t *testing.T, actions []clienttesting.Action) { - testinghelper.AssertEqualNumber(t, len(actions), 2) + testinghelper.AssertEqualNumber(t, len(actions), 4) testinghelper.AssertGet(t, actions[0], "operator.open-cluster-management.io", "v1", "clustermanagers") testinghelper.AssertAction(t, actions[1], "update") - expectedCondition := testinghelper.NamedCondition(registrationDegraded, "UnavailableRegistrationPod", metav1.ConditionTrue) - testinghelper.AssertOnlyConditions(t, actions[1].(clienttesting.UpdateActionImpl).Object, expectedCondition) + expectedCondition1 := testinghelper.NamedCondition(registrationDegraded, "UnavailableRegistrationPod", metav1.ConditionTrue) + testinghelper.AssertOnlyConditions(t, actions[1].(clienttesting.UpdateActionImpl).Object, expectedCondition1) + + testinghelper.AssertGet(t, actions[2], "operator.open-cluster-management.io", "v1", "clustermanagers") + testinghelper.AssertAction(t, actions[3], "update") + expectedCondition2 := testinghelper.NamedCondition(placementDegraded, "PlacementFunctional", metav1.ConditionFalse) + testinghelper.AssertOnlyConditions(t, actions[3].(clienttesting.UpdateActionImpl).Object, expectedCondition1, expectedCondition2) }, }, { - name: "registration functional", + name: "registration functional and no placement deployment", queueKey: testClusterManagerName, clusterManagers: []runtime.Object{newClusterManager()}, - deployments: []runtime.Object{newDeployment(3, 3)}, + deployments: []runtime.Object{newRegistrationDeployment(3, 3)}, validateActions: func(t *testing.T, actions []clienttesting.Action) { - testinghelper.AssertEqualNumber(t, len(actions), 2) + testinghelper.AssertEqualNumber(t, len(actions), 4) testinghelper.AssertGet(t, actions[0], "operator.open-cluster-management.io", "v1", "clustermanagers") testinghelper.AssertAction(t, actions[1], "update") - expectedCondition := testinghelper.NamedCondition(registrationDegraded, "RegistrationFunctional", metav1.ConditionFalse) - testinghelper.AssertOnlyConditions(t, actions[1].(clienttesting.UpdateActionImpl).Object, expectedCondition) + expectedCondition1 := testinghelper.NamedCondition(registrationDegraded, "RegistrationFunctional", metav1.ConditionFalse) + testinghelper.AssertOnlyConditions(t, actions[1].(clienttesting.UpdateActionImpl).Object, expectedCondition1) + + testinghelper.AssertGet(t, actions[2], "operator.open-cluster-management.io", "v1", "clustermanagers") + testinghelper.AssertAction(t, actions[3], "update") + expectedCondition2 := testinghelper.NamedCondition(placementDegraded, "GetPlacementDeploymentFailed", metav1.ConditionTrue) + testinghelper.AssertOnlyConditions(t, actions[3].(clienttesting.UpdateActionImpl).Object, expectedCondition1, expectedCondition2) }, }, } diff --git a/test/e2e/common.go b/test/e2e/common.go index f5be0d891..724062649 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -50,6 +50,7 @@ type Tester struct { hubRegistrationDeployment string hubRegistrationWebhookDeployment string hubWorkWebhookDeployment string + hubPlacementDeployment string operatorNamespace string klusterletOperator string } @@ -67,6 +68,7 @@ func NewTester(kubeconfigPath string) (*Tester, error) { hubRegistrationDeployment: "cluster-manager-registration-controller", hubRegistrationWebhookDeployment: "cluster-manager-registration-webhook", hubWorkWebhookDeployment: "cluster-manager-work-webhook", + hubPlacementDeployment: "cluster-manager-placement-controller", operatorNamespace: "open-cluster-management", klusterletOperator: "klusterlet", } @@ -400,11 +402,33 @@ func (t *Tester) CheckHubReady() error { if _, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). Get(context.TODO(), t.hubWorkWebhookDeployment, metav1.GetOptions{}); err != nil { + } + + if _, err := t.KubeClient.AppsV1().Deployments(t.clusterManagerNamespace). + Get(context.TODO(), t.hubPlacementDeployment, metav1.GetOptions{}); err != nil { return err } return nil } +func (t *Tester) CheckClusterManagerStatus() error { + cms, err := t.OperatorClient.OperatorV1().ClusterManagers().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + if len(cms.Items) == 0 { + return fmt.Errorf("ClusterManager not found") + } + cm := cms.Items[0] + if meta.IsStatusConditionTrue(cm.Status.Conditions, "HubRegistrationDegraded") { + return fmt.Errorf("HubRegistration is degraded") + } + if meta.IsStatusConditionTrue(cm.Status.Conditions, "HubPlacementDegraded") { + return fmt.Errorf("HubPlacement is degraded") + } + return nil +} + func (t *Tester) CheckKlusterletOperatorReady() error { // make sure klusterlet operator deployment is created _, err := t.KubeClient.AppsV1().Deployments(t.operatorNamespace). diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a5b01e56e..bf3f7aaad 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -35,6 +35,10 @@ var _ = BeforeSuite(func() { return t.CheckHubReady() }, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed()) + Eventually(func() error { + return t.CheckClusterManagerStatus() + }, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed()) + Eventually(func() error { return t.CheckKlusterletOperatorReady() }, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())