diff --git a/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller.go b/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller.go index 7c581394b..94642f43e 100644 --- a/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller.go +++ b/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller.go @@ -41,6 +41,7 @@ const ( klusterletWork = "Work" klusterletRegistrationDegraded = "KlusterletRegistrationDegraded" klusterletWorKDegraded = "KlusterletWorkDegraded" + klusterletAvailable = "Available" ) // NewKlusterletStatusController returns a klusterletStatusController @@ -112,9 +113,28 @@ func (k *klusterletStatusController) sync(ctx context.Context, controllerContext []degradedCheckFunc{checkHubConfigSecret, checkAgentDeployment}, ) + availableCondition := checkAgentsDeployment( + ctx, k.kubeClient, + []klusterletAgent{ + { + clusterName: klusterlet.Spec.ClusterName, + deploymentName: fmt.Sprintf("%s-registration-agent", klusterlet.Name), + namespace: klusterletNS, + getSSARFunc: getWorkSelfSubjectAccessReviews, + }, + { + clusterName: klusterlet.Spec.ClusterName, + deploymentName: fmt.Sprintf("%s-work-agent", klusterlet.Name), + namespace: klusterletNS, + getSSARFunc: getWorkSelfSubjectAccessReviews, + }, + }, + ) + _, _, err = helpers.UpdateKlusterletStatus(ctx, k.klusterletClient, klusterletName, helpers.UpdateKlusterletConditionFn(registrationDegradedCondition), helpers.UpdateKlusterletConditionFn(workDegradedCondition), + helpers.UpdateKlusterletConditionFn(availableCondition), ) return err } @@ -126,6 +146,39 @@ type klusterletAgent struct { getSSARFunc getSelfSubjectAccessReviewsFunc } +// Check agent deployments, if both of them have at least 1 available replicas, return available condition +func checkAgentsDeployment(ctx context.Context, kubeClient kubernetes.Interface, agents []klusterletAgent) metav1.Condition { + availableMessages := []string{} + for _, agent := range agents { + deployment, err := kubeClient.AppsV1().Deployments(agent.namespace).Get(ctx, agent.deploymentName, metav1.GetOptions{}) + if err != nil { + return metav1.Condition{ + Type: klusterletAvailable, + Status: metav1.ConditionFalse, + Reason: "GetDeploymentFailed", + Message: fmt.Sprintf("Failed to get deployment %q %q: %v", agent.namespace, agent.deploymentName, err), + } + } + if deployment.Status.AvailableReplicas <= 0 { + return metav1.Condition{ + Type: klusterletAvailable, + Status: metav1.ConditionFalse, + Reason: "NoAvailablePods", + Message: fmt.Sprintf("%v of requested instances are available of deployment %q %q", + deployment.Status.AvailableReplicas, agent.namespace, agent.deploymentName), + } + } + availableMessages = append(availableMessages, agent.deploymentName) + } + + return metav1.Condition{ + Type: klusterletAvailable, + Status: metav1.ConditionTrue, + Reason: "klusterletAvailable", + Message: fmt.Sprintf("deployments are ready: %s", strings.Join(availableMessages, ",")), + } +} + func checkAgentDegradedCondition( ctx context.Context, kubeClient kubernetes.Interface, agentName, degradedType string, diff --git a/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller_test.go b/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller_test.go index 5fae3004d..900b38d95 100644 --- a/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller_test.go +++ b/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller_test.go @@ -172,6 +172,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "BootstrapSecretMissing,HubKubeConfigMissing,GetDeploymentFailed", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "HubKubeConfigMissing,GetDeploymentFailed", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletAvailable, "GetDeploymentFailed", metav1.ConditionFalse), }, }, { @@ -184,6 +185,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "BootstrapSecretError,HubKubeConfigMissing,GetDeploymentFailed", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "HubKubeConfigMissing,GetDeploymentFailed", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletAvailable, "GetDeploymentFailed", metav1.ConditionFalse), }, }, { @@ -196,6 +198,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "BootstrapSecretUnauthorized,HubKubeConfigMissing,GetDeploymentFailed", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "HubKubeConfigMissing,GetDeploymentFailed", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletAvailable, "GetDeploymentFailed", metav1.ConditionFalse), }, }, { @@ -208,6 +211,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "HubKubeConfigSecretMissing,GetDeploymentFailed", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "HubKubeConfigSecretMissing,GetDeploymentFailed", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletAvailable, "GetDeploymentFailed", metav1.ConditionFalse), }, }, { @@ -221,6 +225,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "ClusterNameMissing,GetDeploymentFailed", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "ClusterNameMissing,GetDeploymentFailed", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletAvailable, "GetDeploymentFailed", metav1.ConditionFalse), }, }, { @@ -234,6 +239,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "HubKubeConfigMissing,GetDeploymentFailed", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "HubKubeConfigMissing,GetDeploymentFailed", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletAvailable, "GetDeploymentFailed", metav1.ConditionFalse), }, }, { @@ -247,6 +253,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "HubKubeConfigError,GetDeploymentFailed", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "HubKubeConfigError,GetDeploymentFailed", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletAvailable, "GetDeploymentFailed", metav1.ConditionFalse), }, }, { @@ -260,6 +267,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "HubKubeConfigUnauthorized,GetDeploymentFailed", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "HubKubeConfigUnauthorized,GetDeploymentFailed", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletAvailable, "GetDeploymentFailed", metav1.ConditionFalse), }, }, { @@ -277,6 +285,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "UnavailablePods", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "UnavailablePods", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletAvailable, "NoAvailablePods", metav1.ConditionFalse), }, }, { @@ -294,6 +303,7 @@ func TestSync(t *testing.T) { expectedConditions: []metav1.Condition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "RegistrationFunctional", metav1.ConditionFalse), testinghelper.NamedCondition(klusterletWorKDegraded, "WorkFunctional", metav1.ConditionFalse), + testinghelper.NamedCondition(klusterletAvailable, "klusterletAvailable", metav1.ConditionTrue), }, }, } diff --git a/test/integration/klusterlet_test.go b/test/integration/klusterlet_test.go index 64f166d8e..ec75abd23 100644 --- a/test/integration/klusterlet_test.go +++ b/test/integration/klusterlet_test.go @@ -543,6 +543,46 @@ var _ = ginkgo.Describe("Klusterlet", func() { util.AssertKlusterletCondition(klusterlet.Name, operatorClient, "KlusterletRegistrationDegraded", "RegistrationFunctional", metav1.ConditionFalse) util.AssertKlusterletCondition(klusterlet.Name, operatorClient, "KlusterletWorkDegraded", "WorkFunctional", metav1.ConditionFalse) }) + + ginkgo.It("should have correct available conditions", func() { + _, err := operatorClient.OperatorV1().Klusterlets().Create(context.Background(), klusterlet, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Eventually(func() bool { + if _, err := kubeClient.AppsV1().Deployments(klusterletNamespace).Get(context.Background(), registrationDeploymentName, metav1.GetOptions{}); err != nil { + return false + } + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + + registrationDeployment, err := kubeClient.AppsV1().Deployments(klusterletNamespace).Get(context.Background(), registrationDeploymentName, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Eventually(func() bool { + if _, err := kubeClient.AppsV1().Deployments(klusterletNamespace).Get(context.Background(), workDeploymentName, metav1.GetOptions{}); err != nil { + return false + } + return true + }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue()) + workDeployment, err := kubeClient.AppsV1().Deployments(klusterletNamespace).Get(context.Background(), workDeploymentName, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + util.AssertKlusterletCondition(klusterlet.Name, operatorClient, "Available", "NoAvailablePods", metav1.ConditionFalse) + + // Update replica of deployment, more than 0 AvailableReplicas makes the Available=true + registrationDeployment.Status.AvailableReplicas = 1 + registrationDeployment.Status.Replicas = 3 + registrationDeployment.Status.ReadyReplicas = 3 + _, err = kubeClient.AppsV1().Deployments(klusterletNamespace).UpdateStatus(context.Background(), registrationDeployment, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + workDeployment.Status.AvailableReplicas = 1 + workDeployment.Status.Replicas = 3 + workDeployment.Status.ReadyReplicas = 3 + _, err = kubeClient.AppsV1().Deployments(klusterletNamespace).UpdateStatus(context.Background(), workDeployment, metav1.UpdateOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + util.AssertKlusterletCondition(klusterlet.Name, operatorClient, "Available", "klusterletAvailable", metav1.ConditionTrue) + }) }) ginkgo.Context("bootstrap reconciliation", func() {