From 0b85d55e68043b10f51ca19223d3bdf29e79eec3 Mon Sep 17 00:00:00 2001 From: Amit Singh Date: Wed, 14 Jan 2026 15:58:13 +0530 Subject: [PATCH] Feat: post dispatch output context (#7008) * exploring context data passing Signed-off-by: Amit Singh Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * adds output status fetch logic Signed-off-by: Amit Singh Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * fix: standardize import in dispatcher. Signed-off-by: Chaitanyareddy0702 Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * feat: Allow traits to access workload output status in CUE context Signed-off-by: Chaitanyareddy0702 Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * feat: Implement PostDispatch traits that apply after component health is confirmed. Signed-off-by: Chaitanyareddy0702 Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * feat: Refactor trait handling and status propagation in application dispatch. Signed-off-by: Chaitanyareddy0702 Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * fix: run make reviewable Signed-off-by: Chaitanyareddy0702 Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * feat: Implement and document PostDispatch traits, applying them after component health is confirmed and guarded by a feature flag, along with new example applications. Signed-off-by: Chaitanyareddy0702 Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * feat: Add comments Signed-off-by: Chaitanyareddy0702 Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * Fix: Restore the status field in ctx. Signed-off-by: Vaibhav Agrawal Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * Fix: Error for evaluating the status of the trait Signed-off-by: Chaitanyareddy0702 Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * refactor: removes minor unnecessary changes Signed-off-by: Amit Singh Signed-off-by: semmet95 Signed-off-by: Vishal Kumar * refactor: minor linter changes Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * test: Add comprehensive tests for PostDispatch traits and their status handling Signed-off-by: Reetika Malhotra Signed-off-by: Vishal Kumar * Fix: Increase multi-cluster test time Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * Chore: Add focus and print the application status Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * Chore: print deployment status in the multicluster test Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * Chore: add labels for the deployment Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * debugging test failure Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * debugging test failure by updating multi cluster ctx Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * undoes multi cluster ctx change Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * Feat: enable MultiStageComponentApply feature by default Signed-off-by: Chaitanya Reddy Onteddu Signed-off-by: Vishal Kumar * Feat: implement post-dispatch traits application in workflow states Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * Chore: remove unnecessary blank lines in application_controller.go Signed-off-by: Amit Singh Signed-off-by: Vishal Kumar * Feat: enhance output readiness handling in health checks Signed-off-by: Vishal Kumar * Feat: add logic to determine need for post-dispatch outputs in workload processing Signed-off-by: Vishal Kumar * Feat: enhance output extraction and dependency checking for post-dispatch traits Signed-off-by: Vishal Kumar * fix code to exclude validation of post dispatch trait in webhook Signed-off-by: Vishal Kumar * fix code to exclude validation of post dispatch trait in webhook Signed-off-by: Vishal Kumar * commit for running the test again Signed-off-by: Vishal Kumar * commit for running the test again Signed-off-by: Vishal Kumar * commit for running the test again Signed-off-by: Vishal Kumar * triggering checks Signed-off-by: Amit Singh * chore: adds explanation comments Signed-off-by: Amit Singh * chore: adds errors to context Signed-off-by: Amit Singh * chore: minor improvements Signed-off-by: Amit Singh * fix: update output handling for pending PostDispatch traits Signed-off-by: Vishal Kumar * fix: improve output handling for PostDispatch traits in deploy process Signed-off-by: Vishal Kumar * fix: streamline output handling in PostDispatch process Signed-off-by: Vishal Kumar * chore: commit to re run the pipeline Signed-off-by: Vishal Kumar * chore: commit to re run the pipeline Signed-off-by: Vishal Kumar * chore: commit to re run the pipeline Signed-off-by: Vishal Kumar * fix: enhance output status handling in PostDispatch context for multi-stage support Signed-off-by: Vishal Kumar * chore: commit to re run the pipeline Signed-off-by: Vishal Kumar * fix: increase timeout for PostDispatch trait verification in tests Signed-off-by: Vishal Kumar * fix: enhance output status handling in PostDispatch context for multi-stage support Signed-off-by: Vishal Kumar * chore: commit to re run the pipeline Signed-off-by: Vishal Kumar --------- Signed-off-by: Amit Singh Signed-off-by: semmet95 Signed-off-by: Vishal Kumar Signed-off-by: Chaitanyareddy0702 Signed-off-by: Vaibhav Agrawal Signed-off-by: Reetika Malhotra Signed-off-by: Chaitanya Reddy Onteddu Co-authored-by: Chitanya Reddy Onteddu Co-authored-by: Vishal Kumar --- pkg/appfile/validate.go | 8 + .../application/application_controller.go | 28 + .../core.oam.dev/v1beta1/application/apply.go | 126 +- .../v1beta1/application/generator.go | 75 ++ pkg/cue/definition/template.go | 69 +- pkg/cue/process/handle.go | 4 + pkg/features/controller_features.go | 2 +- pkg/oam/util/helper.go | 42 + .../providers/legacy/multicluster/deploy.go | 31 +- pkg/workflow/providers/multicluster/deploy.go | 31 +- .../multicluster_test.go | 9 +- test/e2e-test/postdispatch_trait_test.go | 1105 +++++++++++++++++ 12 files changed, 1513 insertions(+), 17 deletions(-) create mode 100644 test/e2e-test/postdispatch_trait_test.go diff --git a/pkg/appfile/validate.go b/pkg/appfile/validate.go index 84c0d2492..380ee996e 100644 --- a/pkg/appfile/validate.go +++ b/pkg/appfile/validate.go @@ -27,6 +27,8 @@ import ( "github.com/kubevela/workflow/pkg/cue/model/value" utilfeature "k8s.io/apiserver/pkg/util/feature" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/pkg/features" "github.com/pkg/errors" @@ -64,6 +66,12 @@ func (p *Parser) ValidateCUESchematicAppfile(a *Appfile) error { if tr.CapabilityCategory != types.CUECategory { continue } + if tr.FullTemplate != nil && + tr.FullTemplate.TraitDefinition.Spec.Stage == v1beta1.PostDispatch { + // PostDispatch type trait validation at this point might fail as they could have + // references to fields that are populated/injected during runtime only + continue + } if err := tr.EvalContext(pCtx); err != nil { return errors.WithMessagef(err, "cannot evaluate trait %q", tr.Name) } diff --git a/pkg/controller/core.oam.dev/v1beta1/application/application_controller.go b/pkg/controller/core.oam.dev/v1beta1/application/application_controller.go index 697ebd999..94c583fad 100644 --- a/pkg/controller/core.oam.dev/v1beta1/application/application_controller.go +++ b/pkg/controller/core.oam.dev/v1beta1/application/application_controller.go @@ -222,8 +222,25 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu workflowInstance.Status.Phase = workflowState app.Status.Workflow = workflow.ConvertWorkflowStatus(workflowInstance.Status, app.Status.Workflow.AppRevision) logCtx.Info(fmt.Sprintf("Workflow return state=%s", workflowState)) + postDispatchApplied := false + applyPostDispatchTraits := func() error { + if postDispatchApplied || !feature.DefaultMutableFeatureGate.Enabled(features.MultiStageComponentApply) { + return nil + } + if err := handler.applyPostDispatchTraits(logCtx, appParser, appFile); err != nil { + logCtx.Error(err, "Failed to apply PostDispatch traits") + r.Recorder.Event(app, event.Warning(velatypes.ReasonFailedApply, err)) + return err + } + app.Status.AppliedResources = handler.appliedResources + postDispatchApplied = true + return nil + } switch workflowState { case workflowv1alpha1.WorkflowStateSuspending: + if err := applyPostDispatchTraits(); err != nil { + return r.endWithNegativeCondition(logCtx, app, condition.ReconcileError(err), common.ApplicationWorkflowSuspending) + } if duration := workflowExecutor.GetSuspendBackoffWaitTime(); duration > 0 { _, err = r.gcResourceTrackers(logCtx, handler, common.ApplicationWorkflowSuspending, false, workflowUpdated) return r.result(err).requeue(duration).ret() @@ -243,6 +260,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } return r.gcResourceTrackers(logCtx, handler, common.ApplicationWorkflowFailed, false, workflowUpdated) case workflowv1alpha1.WorkflowStateExecuting: + if err := applyPostDispatchTraits(); err != nil { + return r.endWithNegativeCondition(logCtx, app, condition.ReconcileError(err), common.ApplicationRunningWorkflow) + } _, err = r.gcResourceTrackers(logCtx, handler, common.ApplicationRunningWorkflow, false, workflowUpdated) return r.result(err).requeue(workflowExecutor.GetBackoffWaitTime()).ret() case workflowv1alpha1.WorkflowStateSucceeded: @@ -250,6 +270,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu r.doWorkflowFinish(logCtx, app, handler, workflowState) } case workflowv1alpha1.WorkflowStateSkipped: + if err := applyPostDispatchTraits(); err != nil { + return r.endWithNegativeCondition(logCtx, app, condition.ReconcileError(err), common.ApplicationRunningWorkflow) + } return r.result(nil).requeue(workflowExecutor.GetBackoffWaitTime()).ret() default: } @@ -260,6 +283,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu phase = common.ApplicationUnhealthy } + // Apply PostDispatch traits for healthy components if not already done in workflow requeue branch + if err := applyPostDispatchTraits(); err != nil { + return r.endWithNegativeCondition(logCtx, app, condition.ReconcileError(err), phase) + } + r.stateKeep(logCtx, handler, app) opts := []resourcekeeper.GCOption{ diff --git a/pkg/controller/core.oam.dev/v1beta1/application/apply.go b/pkg/controller/core.oam.dev/v1beta1/application/apply.go index a6f11c898..59adb7495 100644 --- a/pkg/controller/core.oam.dev/v1beta1/application/apply.go +++ b/pkg/controller/core.oam.dev/v1beta1/application/apply.go @@ -36,6 +36,7 @@ import ( "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/appfile" + velaprocess "github.com/oam-dev/kubevela/pkg/cue/process" "github.com/oam-dev/kubevela/pkg/monitor/metrics" "github.com/oam-dev/kubevela/pkg/multicluster" "github.com/oam-dev/kubevela/pkg/oam" @@ -430,9 +431,130 @@ func extractOutputAndOutputs(templateContext map[string]interface{}) (*unstructu func extractOutputs(templateContext map[string]interface{}) []*unstructured.Unstructured { outputs := make([]*unstructured.Unstructured, 0) if templateContext["outputs"] != nil { - for _, v := range templateContext["outputs"].(map[string]interface{}) { - outputs = append(outputs, &unstructured.Unstructured{Object: v.(map[string]interface{})}) + for k, v := range templateContext["outputs"].(map[string]interface{}) { + obj := &unstructured.Unstructured{Object: v.(map[string]interface{})} + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + if labels[oam.TraitResource] == "" && k != "" { + labels[oam.TraitResource] = k + } + obj.SetLabels(labels) + outputs = append(outputs, obj) } } return outputs } + +// applyPostDispatchTraits applies PostDispatch stage traits for healthy components. +// This is called after the workflow succeeds and component health is confirmed. +func (h *AppHandler) applyPostDispatchTraits(ctx monitorContext.Context, appParser *appfile.Parser, af *appfile.Appfile) error { + for _, svc := range h.services { + if !svc.Healthy { + continue + } + + // Find the component spec + var comp common.ApplicationComponent + found := false + for _, c := range h.app.Spec.Components { + if c.Name == svc.Name { + comp = c + found = true + break + } + } + if !found { + continue + } + + // Parse the component to get all traits + wl, err := appParser.ParseComponentFromRevisionAndClient(ctx.GetContext(), comp, h.currentAppRev) + if err != nil { + return errors.WithMessagef(err, "failed to parse component %s for PostDispatch traits", comp.Name) + } + + // Filter to keep ONLY PostDispatch traits + var postDispatchTraits []*appfile.Trait + for _, trait := range wl.Traits { + if trait.FullTemplate.TraitDefinition.Spec.Stage == v1beta1.PostDispatch { + postDispatchTraits = append(postDispatchTraits, trait) + } + } + + if len(postDispatchTraits) == 0 { + continue + } + + wl.Traits = postDispatchTraits + + // Generate manifest with context that includes live workload status + manifest, err := af.GenerateComponentManifest(wl, func(ctxData *velaprocess.ContextData) { + if svc.Namespace != "" { + ctxData.Namespace = svc.Namespace + } + if svc.Cluster != "" { + ctxData.Cluster = svc.Cluster + } else { + ctxData.Cluster = pkgmulticluster.Local + } + ctxData.ClusterVersion = multicluster.GetVersionInfoFromObject( + pkgmulticluster.WithCluster(ctx.GetContext(), types.ClusterLocalName), + h.Client, + ctxData.Cluster, + ) + + // Fetch live workload status for PostDispatch traits to use if it's created on the cluster + tempCtx := appfile.NewBasicContext(*ctxData, wl.Params) + if err := wl.EvalContext(tempCtx); err != nil { + ctx.Error(err, "failed to evaluate context for workload %s", wl.Name) + return + } + base, _ := tempCtx.Output() + componentWorkload, err := base.Unstructured() + if err != nil { + ctx.Error(err, "failed to unstructure base component generated using workload %s", wl.Name) + return + } + if componentWorkload.GetName() == "" { + componentWorkload.SetName(ctxData.CompName) + } + _ctx := util.WithCluster(tempCtx.GetCtx(), componentWorkload) + object, err := util.GetResourceFromObj(_ctx, tempCtx, componentWorkload, h.Client, ctxData.Namespace, map[string]string{ + oam.LabelOAMResourceType: oam.ResourceTypeWorkload, + oam.LabelAppComponent: ctxData.CompName, + oam.LabelAppName: ctxData.AppName, + }, "") + if err != nil { + ctx.Error(err, "failed to fetch workload output resource %s from the cluster", componentWorkload.GetName()) + return + } + ctxData.Output = object + }) + if err != nil { + return errors.WithMessagef(err, "failed to generate manifest for PostDispatch traits of component %s", comp.Name) + } + + // Render traits + _, readyTraits, err := renderComponentsAndTraits(manifest, h.currentAppRev, svc.Cluster, svc.Namespace) + if err != nil { + return errors.WithMessagef(err, "failed to render PostDispatch traits for component %s", comp.Name) + } + + // Add app ownership labels + for _, trait := range readyTraits { + util.AddLabels(trait, map[string]string{ + oam.LabelAppName: h.app.GetName(), + oam.LabelAppNamespace: h.app.GetNamespace(), + }) + } + + // Dispatch the traits + dispatchCtx := multicluster.ContextWithClusterName(ctx.GetContext(), svc.Cluster) + if err := h.Dispatch(dispatchCtx, h.Client, svc.Cluster, common.WorkflowResourceCreator, readyTraits...); err != nil { + return errors.WithMessagef(err, "failed to dispatch PostDispatch traits for component %s", comp.Name) + } + } + return nil +} diff --git a/pkg/controller/core.oam.dev/v1beta1/application/generator.go b/pkg/controller/core.oam.dev/v1beta1/application/generator.go index 138d5dc57..057182a7a 100644 --- a/pkg/controller/core.oam.dev/v1beta1/application/generator.go +++ b/pkg/controller/core.oam.dev/v1beta1/application/generator.go @@ -430,6 +430,30 @@ func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context, return nil, nil, errors.WithMessage(err, "ParseWorkload") } wl.Patch = patcher + + // Add all traits to the workload if MultiStageComponentApply is disabled + if utilfeature.DefaultMutableFeatureGate.Enabled(features.MultiStageComponentApply) { + serviceHealthy := false + needPostDispatchOutputs := componentOutputsConsumed(comp, af.Components) + for _, svc := range h.services { + if svc.Name == comp.Name { + serviceHealthy = svc.Healthy + break + } + } + // not including PostDispatch type traits in the workload if the component service is not healthy + // because PostDispatch type traits might have references to fields that are only populated when the service is healthy + if !serviceHealthy && !needPostDispatchOutputs { + nonPostDispatchTraits := []*appfile.Trait{} + for _, trait := range wl.Traits { + if trait.FullTemplate.TraitDefinition.Spec.Stage != v1beta1.PostDispatch { + nonPostDispatchTraits = append(nonPostDispatchTraits, trait) + } + } + wl.Traits = nonPostDispatchTraits + } + } + manifest, err := af.GenerateComponentManifest(wl, func(ctxData *velaprocess.ContextData) { if ns := componentNamespaceFromContext(ctx); ns != "" { ctxData.Namespace = ns @@ -444,6 +468,32 @@ func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context, // cluster info are secrets stored in the control plane cluster ctxData.ClusterVersion = multicluster.GetVersionInfoFromObject(pkgmulticluster.WithCluster(ctx, types.ClusterLocalName), h.Client, ctxData.Cluster) ctxData.CompRevision, _ = ctrlutil.ComputeSpecHash(comp) + + if utilfeature.DefaultMutableFeatureGate.Enabled(features.MultiStageComponentApply) { + // inject the main workload output as "output" in the context + tempCtx := appfile.NewBasicContext(*ctxData, wl.Params) + if err := wl.EvalContext(tempCtx); err != nil { + return + } + base, _ := tempCtx.Output() + componentWorkload, err := base.Unstructured() + if err != nil { + return + } + if componentWorkload.GetName() == "" { + componentWorkload.SetName(ctxData.CompName) + } + _ctx := util.WithCluster(tempCtx.GetCtx(), componentWorkload) + object, err := util.GetResourceFromObj(_ctx, tempCtx, componentWorkload, h.Client, ctxData.Namespace, map[string]string{ + oam.LabelOAMResourceType: oam.ResourceTypeWorkload, + oam.LabelAppComponent: ctxData.CompName, + oam.LabelAppName: ctxData.AppName, + }, "") + if err != nil { + return + } + ctxData.Output = object + } }) if err != nil { return nil, nil, errors.WithMessage(err, "GenerateComponentManifest") @@ -475,6 +525,31 @@ func renderComponentsAndTraits(manifest *types.ComponentManifest, appRev *v1beta return readyWorkload, readyTraits, nil } +// componentOutputsConsumed returns true if any other component depends on outputs produced +// from PostDispatch traits (valueFrom starting with "outputs."). +func componentOutputsConsumed(comp common.ApplicationComponent, components []common.ApplicationComponent) bool { + outputNames := map[string]struct{}{} + for _, o := range comp.Outputs { + if strings.HasPrefix(o.ValueFrom, "outputs.") { + outputNames[o.Name] = struct{}{} + } + } + if len(outputNames) == 0 { + return false + } + for _, c := range components { + if c.Name == comp.Name { + continue + } + for _, in := range c.Inputs { + if _, ok := outputNames[in.From]; ok { + return true + } + } + } + return false +} + func checkSkipApplyWorkload(comp *appfile.Component) { for _, trait := range comp.Traits { if trait.FullTemplate.TraitDefinition.Spec.ManageWorkload { diff --git a/pkg/cue/definition/template.go b/pkg/cue/definition/template.go index 740e98aed..4cbe17535 100644 --- a/pkg/cue/definition/template.go +++ b/pkg/cue/definition/template.go @@ -23,7 +23,10 @@ import ( "sort" "strings" + "k8s.io/apiserver/pkg/util/feature" + "github.com/oam-dev/kubevela/pkg/cue/definition/health" + "github.com/oam-dev/kubevela/pkg/features" "github.com/kubevela/pkg/cue/cuex" @@ -261,10 +264,24 @@ func (td *traitDef) Complete(ctx process.Context, abstractTemplate string, param buff += fmt.Sprintf("%s: %s\n", velaprocess.ParameterFieldName, string(bt)) } } + + multiStageEnabled := feature.DefaultMutableFeatureGate.Enabled(features.MultiStageComponentApply) + var statusBytes []byte + if multiStageEnabled { + statusBytes = outputStatusBytes(ctx) + } + c, err := ctx.BaseContextFile() if err != nil { return err } + + // When multi-stage is enabled, merge the existing output.status from ctx into the + // base context so downstream CUE can reference it deterministically. + if multiStageEnabled { + c = injectOutputStatusIntoBaseContext(ctx, c, statusBytes) + } + buff += c val, err := cuex.DefaultCompiler.Get().CompileString(ctx.GetCtx(), buff) @@ -341,6 +358,56 @@ func (td *traitDef) Complete(ctx process.Context, abstractTemplate string, param return nil } +func outputStatusBytes(ctx process.Context) []byte { + var statusBytes []byte + var outputMap map[string]interface{} + if output := ctx.GetData(OutputFieldName); output != nil { + if m, ok := output.(map[string]interface{}); ok { + outputMap = m + } else if ptr, ok := output.(*interface{}); ok && ptr != nil { + if m, ok := (*ptr).(map[string]interface{}); ok { + outputMap = m + } + } + + if outputMap != nil { + if status, ok := outputMap["status"]; ok { + if b, err := json.Marshal(status); err == nil { + statusBytes = b + } + } + } + } + return statusBytes +} + +func injectOutputStatusIntoBaseContext(ctx process.Context, c string, statusBytes []byte) string { + if len(statusBytes) > 0 { + // If output is an empty object, replace it with only the status field without trailing comma. + emptyOutputMarker := "\"output\":{}" + if strings.Contains(c, emptyOutputMarker) { + replacement := fmt.Sprintf("\"output\":{\"status\":%s}", string(statusBytes)) + c = strings.Replace(c, emptyOutputMarker, replacement, 1) + } else { + // Otherwise, insert status as the first field and keep the comma to separate from existing fields. + replacement := fmt.Sprintf("\"output\":{\"status\":%s,", string(statusBytes)) + c = strings.Replace(c, "\"output\":{", replacement, 1) + } + + // Restore the status field to the current output in ctx.data + var status interface{} + if err := json.Unmarshal(statusBytes, &status); err == nil { + if currentOutput := ctx.GetData(OutputFieldName); currentOutput != nil { + if currentMap, ok := currentOutput.(map[string]interface{}); ok { + currentMap["status"] = status + ctx.PushData(OutputFieldName, currentMap) + } + } + } + } + return c +} + func parseErrors(errs cue.Value) error { if it, e := errs.List(); e == nil { for it.Next() { @@ -399,8 +466,8 @@ func (td *traitDef) getTemplateContext(ctx process.Context, cli client.Reader, a baseLabels := GetBaseContextLabels(ctx) var root = initRoot(baseLabels) var commonLabels = GetCommonLabels(baseLabels) - _, assists := ctx.Output() + outputs := make(map[string]interface{}) for _, assist := range assists { if assist.Type != td.name { diff --git a/pkg/cue/process/handle.go b/pkg/cue/process/handle.go index 2b4b64284..61da8381a 100644 --- a/pkg/cue/process/handle.go +++ b/pkg/cue/process/handle.go @@ -50,6 +50,7 @@ type ContextData struct { AppAnnotations map[string]string ClusterVersion types.ClusterVersion + Output interface{} } // NewContext creates a new process context @@ -75,6 +76,9 @@ func NewContext(data ContextData) process.Context { ctx.PushData(ContextAppRevisionNum, revNum) ctx.PushData(ContextCluster, data.Cluster) ctx.PushData(ContextClusterVersion, parseClusterVersion(data.ClusterVersion)) + if data.Output != nil { + ctx.PushData(OutputFieldName, data.Output) + } return ctx } diff --git a/pkg/features/controller_features.go b/pkg/features/controller_features.go index 16a8518f3..2e3970c63 100644 --- a/pkg/features/controller_features.go +++ b/pkg/features/controller_features.go @@ -138,7 +138,7 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ GzipResourceTracker: {Default: false, PreRelease: featuregate.Alpha}, ZstdResourceTracker: {Default: false, PreRelease: featuregate.Alpha}, ApplyOnce: {Default: false, PreRelease: featuregate.Alpha}, - MultiStageComponentApply: {Default: false, PreRelease: featuregate.Alpha}, + MultiStageComponentApply: {Default: true, PreRelease: featuregate.Alpha}, GzipApplicationRevision: {Default: false, PreRelease: featuregate.Alpha}, ZstdApplicationRevision: {Default: false, PreRelease: featuregate.Alpha}, PreDispatchDryRun: {Default: true, PreRelease: featuregate.Alpha}, diff --git a/pkg/oam/util/helper.go b/pkg/oam/util/helper.go index 462144607..fa9d4a79a 100644 --- a/pkg/oam/util/helper.go +++ b/pkg/oam/util/helper.go @@ -27,6 +27,9 @@ import ( "strings" "github.com/davecgh/go-spew/spew" + "github.com/kubevela/pkg/multicluster" + "github.com/kubevela/workflow/pkg/cue/model" + "github.com/kubevela/workflow/pkg/cue/process" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -835,3 +838,42 @@ func (accessor *applicationResourceNamespaceAccessor) Namespace() string { func NewApplicationResourceNamespaceAccessor(appNs, overrideNs string) NamespaceAccessor { return &applicationResourceNamespaceAccessor{applicationNamespace: appNs, overrideNamespace: overrideNs} } + +func WithCluster(ctx context.Context, o client.Object) context.Context { + if cluster := oam.GetCluster(o); cluster != "" { + return multicluster.WithCluster(ctx, cluster) + } + return ctx +} + +func GetResourceFromObj(ctx context.Context, pctx process.Context, obj *unstructured.Unstructured, client client.Reader, namespace string, labels map[string]string, outputsResource string) (map[string]interface{}, error) { + if outputsResource != "" { + labels[oam.TraitResource] = outputsResource + } + if obj.GetName() != "" { + u, err := GetObjectGivenGVKAndName(ctx, client, obj.GroupVersionKind(), namespace, obj.GetName()) + if err != nil { + return nil, err + } + return u.Object, nil + } + if ctxName, ok := pctx.GetData(model.ContextName).(string); ok && ctxName != "" { + u, err := GetObjectGivenGVKAndName(ctx, client, obj.GroupVersionKind(), namespace, ctxName) + if err == nil { + return u.Object, nil + } + } + list, err := GetObjectsGivenGVKAndLabels(ctx, client, obj.GroupVersionKind(), namespace, labels) + if err != nil { + return nil, err + } + if len(list.Items) == 1 { + return list.Items[0].Object, nil + } + for _, v := range list.Items { + if v.GetLabels()[oam.TraitResource] == outputsResource { + return v.Object, nil + } + } + return nil, errors.Errorf("no resources found gvk(%v) labels(%v)", obj.GroupVersionKind(), labels) +} diff --git a/pkg/workflow/providers/legacy/multicluster/deploy.go b/pkg/workflow/providers/legacy/multicluster/deploy.go index 0fbf9c1fc..d3cbf6e5c 100644 --- a/pkg/workflow/providers/legacy/multicluster/deploy.go +++ b/pkg/workflow/providers/legacy/multicluster/deploy.go @@ -296,9 +296,12 @@ type applyTaskResult struct { healthy bool err error task *applyTask + // outputReady indicates whether all declared outputs are ready + outputReady bool } // applyComponents will apply components to placements. +// nolint:gocyclo func applyComponents(ctx context.Context, apply oamprovidertypes.ComponentApply, healthCheck oamprovidertypes.ComponentHealthCheck, components []common.ApplicationComponent, placements []v1alpha1.PlacementDecision, parallelism int) (bool, string, error) { var tasks []*applyTask var cache = pkgmaps.NewSyncMap[string, cue.Value]() @@ -321,6 +324,8 @@ func applyComponents(ctx context.Context, apply oamprovidertypes.ComponentApply, } unhealthyResults := make([]*applyTaskResult, 0) maxHealthCheckTimes := len(tasks) + outputNotReadyReasons := make([]string, 0) + outputsReady := true HealthCheck: for i := 0; i < maxHealthCheckTimes; i++ { checkTasks := make([]*applyTask, 0) @@ -343,13 +348,25 @@ HealthCheck: healthy, _, output, outputs, err := healthCheck(ctx, task.component, nil, task.placement.Cluster, task.placement.Namespace) task.healthy = ptr.To(healthy) if healthy { - err = task.generateOutput(output, outputs, cache, makeValue) + if errOutput := task.generateOutput(output, outputs, cache, makeValue); errOutput != nil { + var notFound workflowerrors.LookUpNotFoundErr + if errors.As(errOutput, ¬Found) && strings.HasPrefix(string(notFound), "outputs.") && len(outputs) == 0 { + // PostDispatch traits are not rendered/applied yet, so trait outputs are unavailable. + // Skip blocking the deploy step; the outputs will be populated after PostDispatch runs. + errOutput = nil + } + err = errOutput + } } - return &applyTaskResult{healthy: healthy, err: err, task: task} + return &applyTaskResult{healthy: healthy, err: err, task: task, outputReady: true} }, slices.Parallelism(parallelism)) for _, res := range checkResults { taskHealthyMap[res.task.key()] = res.healthy + if !res.outputReady { + outputsReady = false + outputNotReadyReasons = append(outputNotReadyReasons, fmt.Sprintf("%s outputs not ready", res.task.key())) + } if !res.healthy || res.err != nil { unhealthyResults = append(unhealthyResults, res) } @@ -374,13 +391,13 @@ HealthCheck: results = slices.ParMap[*applyTask, *applyTaskResult](todoTasks, func(task *applyTask) *applyTaskResult { err := task.fillInputs(cache, makeValue) if err != nil { - return &applyTaskResult{healthy: false, err: err, task: task} + return &applyTaskResult{healthy: false, err: err, task: task, outputReady: true} } _, _, healthy, err := apply(ctx, task.component, nil, task.placement.Cluster, task.placement.Namespace) if err != nil { - return &applyTaskResult{healthy: healthy, err: err, task: task} + return &applyTaskResult{healthy: healthy, err: err, task: task, outputReady: true} } - return &applyTaskResult{healthy: healthy, err: err, task: task} + return &applyTaskResult{healthy: healthy, err: err, task: task, outputReady: true} }, slices.Parallelism(parallelism)) } var errs []error @@ -401,11 +418,13 @@ HealthCheck: } } + reasons = append(reasons, outputNotReadyReasons...) + for _, t := range pendingTasks { reasons = append(reasons, fmt.Sprintf("%s is waiting dependents", t.key())) } - return allHealthy && len(pendingTasks) == 0, strings.Join(reasons, ","), velaerrors.AggregateErrors(errs) + return allHealthy && outputsReady && len(pendingTasks) == 0, strings.Join(reasons, ","), velaerrors.AggregateErrors(errs) } func fieldPathToComponent(input string) string { diff --git a/pkg/workflow/providers/multicluster/deploy.go b/pkg/workflow/providers/multicluster/deploy.go index 0fbf9c1fc..d3cbf6e5c 100644 --- a/pkg/workflow/providers/multicluster/deploy.go +++ b/pkg/workflow/providers/multicluster/deploy.go @@ -296,9 +296,12 @@ type applyTaskResult struct { healthy bool err error task *applyTask + // outputReady indicates whether all declared outputs are ready + outputReady bool } // applyComponents will apply components to placements. +// nolint:gocyclo func applyComponents(ctx context.Context, apply oamprovidertypes.ComponentApply, healthCheck oamprovidertypes.ComponentHealthCheck, components []common.ApplicationComponent, placements []v1alpha1.PlacementDecision, parallelism int) (bool, string, error) { var tasks []*applyTask var cache = pkgmaps.NewSyncMap[string, cue.Value]() @@ -321,6 +324,8 @@ func applyComponents(ctx context.Context, apply oamprovidertypes.ComponentApply, } unhealthyResults := make([]*applyTaskResult, 0) maxHealthCheckTimes := len(tasks) + outputNotReadyReasons := make([]string, 0) + outputsReady := true HealthCheck: for i := 0; i < maxHealthCheckTimes; i++ { checkTasks := make([]*applyTask, 0) @@ -343,13 +348,25 @@ HealthCheck: healthy, _, output, outputs, err := healthCheck(ctx, task.component, nil, task.placement.Cluster, task.placement.Namespace) task.healthy = ptr.To(healthy) if healthy { - err = task.generateOutput(output, outputs, cache, makeValue) + if errOutput := task.generateOutput(output, outputs, cache, makeValue); errOutput != nil { + var notFound workflowerrors.LookUpNotFoundErr + if errors.As(errOutput, ¬Found) && strings.HasPrefix(string(notFound), "outputs.") && len(outputs) == 0 { + // PostDispatch traits are not rendered/applied yet, so trait outputs are unavailable. + // Skip blocking the deploy step; the outputs will be populated after PostDispatch runs. + errOutput = nil + } + err = errOutput + } } - return &applyTaskResult{healthy: healthy, err: err, task: task} + return &applyTaskResult{healthy: healthy, err: err, task: task, outputReady: true} }, slices.Parallelism(parallelism)) for _, res := range checkResults { taskHealthyMap[res.task.key()] = res.healthy + if !res.outputReady { + outputsReady = false + outputNotReadyReasons = append(outputNotReadyReasons, fmt.Sprintf("%s outputs not ready", res.task.key())) + } if !res.healthy || res.err != nil { unhealthyResults = append(unhealthyResults, res) } @@ -374,13 +391,13 @@ HealthCheck: results = slices.ParMap[*applyTask, *applyTaskResult](todoTasks, func(task *applyTask) *applyTaskResult { err := task.fillInputs(cache, makeValue) if err != nil { - return &applyTaskResult{healthy: false, err: err, task: task} + return &applyTaskResult{healthy: false, err: err, task: task, outputReady: true} } _, _, healthy, err := apply(ctx, task.component, nil, task.placement.Cluster, task.placement.Namespace) if err != nil { - return &applyTaskResult{healthy: healthy, err: err, task: task} + return &applyTaskResult{healthy: healthy, err: err, task: task, outputReady: true} } - return &applyTaskResult{healthy: healthy, err: err, task: task} + return &applyTaskResult{healthy: healthy, err: err, task: task, outputReady: true} }, slices.Parallelism(parallelism)) } var errs []error @@ -401,11 +418,13 @@ HealthCheck: } } + reasons = append(reasons, outputNotReadyReasons...) + for _, t := range pendingTasks { reasons = append(reasons, fmt.Sprintf("%s is waiting dependents", t.key())) } - return allHealthy && len(pendingTasks) == 0, strings.Join(reasons, ","), velaerrors.AggregateErrors(errs) + return allHealthy && outputsReady && len(pendingTasks) == 0, strings.Join(reasons, ","), velaerrors.AggregateErrors(errs) } func fieldPathToComponent(input string) string { diff --git a/test/e2e-multicluster-test/multicluster_test.go b/test/e2e-multicluster-test/multicluster_test.go index 6f7d26575..e5a752ddd 100644 --- a/test/e2e-multicluster-test/multicluster_test.go +++ b/test/e2e-multicluster-test/multicluster_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "os" + "os/exec" "strings" "time" @@ -478,7 +479,12 @@ var _ = Describe("Test multicluster scenario", func() { Eventually(func(g Gomega) { g.Expect(k8sClient.Get(hubCtx, client.ObjectKeyFromObject(app), app)).Should(Succeed()) g.Expect(app.Status.Phase).Should(Equal(common.ApplicationRunning)) - }, 20*time.Second).Should(Succeed()) + }, 10*time.Minute).Should(Succeed()) + + By("print application status") + out, err := exec.Command("kubectl", "describe", "application", app.Name, "-n", testNamespace).CombinedOutput() + Expect(err).Should(Succeed()) + fmt.Println(string(out)) By("test dispatched resource") svc := &corev1.Service{} @@ -489,6 +495,7 @@ var _ = Describe("Test multicluster scenario", func() { Expect(cm.Data["host"]).Should(Equal(host)) Expect(k8sClient.Get(workerCtx, client.ObjectKey{Namespace: testNamespace, Name: app.Name}, cm)).Should(Succeed()) Expect(cm.Data["host"]).Should(Equal(host)) + }) It("Test application with workflow change will rerun", func() { diff --git a/test/e2e-test/postdispatch_trait_test.go b/test/e2e-test/postdispatch_trait_test.go new file mode 100644 index 000000000..5c38cf24d --- /dev/null +++ b/test/e2e-test/postdispatch_trait_test.go @@ -0,0 +1,1105 @@ +/* +Copyright 2025 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/common" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" +) + +var _ = Describe("PostDispatch Trait tests", func() { + ctx := context.Background() + var namespace string + + BeforeEach(func() { + namespace = randomNamespaceName("postdispatch-test") + Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}})).Should(Succeed()) + }) + + AfterEach(func() { + By("Cleaning up test namespace") + ns := &corev1.Namespace{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: namespace}, ns)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, ns)).Should(Succeed()) + }) + + Context("Test PostDispatch status for trait, component and application", func() { + It("Should mark application, component, and PostDispatch traits healthy", func() { + deploymentTraitName := "test-deployment-trait-" + randomNamespaceName("") + cmTraitName := "test-cm-trait-" + randomNamespaceName("") + + By("Creating PostDispatch deployment trait definition") + deploymentTrait := &v1beta1.TraitDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentTraitName, + Namespace: "vela-system", + }, + Spec: v1beta1.TraitDefinitionSpec{ + Stage: v1beta1.PostDispatch, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +outputs: statusPod: { + apiVersion: "apps/v1" + kind: "Deployment" + metadata: { + name: parameter.name + } + spec: { + replicas: context.output.status.replicas + selector: matchLabels: { + app: parameter.name + } + template: { + metadata: labels: { + app: parameter.name + } + spec: containers: [{ + name: parameter.name + image: parameter.image + }] + } + } +} + +parameter: { + name: string + image: string +} +`, + }, + }, + Status: &common.Status{ + HealthPolicy: `pod: context.outputs.statusPod +ready: { + updatedReplicas: *0 | int + readyReplicas: *0 | int + replicas: *0 | int + observedGeneration: *0 | int +} & { + if pod.status.updatedReplicas != _|_ { + updatedReplicas: pod.status.updatedReplicas + } + if pod.status.readyReplicas != _|_ { + readyReplicas: pod.status.readyReplicas + } + if pod.status.replicas != _|_ { + replicas: pod.status.replicas + } + if pod.status.observedGeneration != _|_ { + observedGeneration: pod.status.observedGeneration + } +} +_isHealth: (pod.spec.replicas == ready.readyReplicas) && (pod.spec.replicas == ready.updatedReplicas) && (pod.spec.replicas == ready.replicas) && (ready.observedGeneration == pod.metadata.generation || ready.observedGeneration > pod.metadata.generation) +isHealth: *_isHealth | bool +if pod.metadata.annotations != _|_ { + if pod.metadata.annotations["app.oam.dev/disable-health-check"] != _|_ { + isHealth: true + } +} +`, + }, + }, + } + Expect(k8sClient.Create(ctx, deploymentTrait)).Should(Succeed()) + + By("Creating PostDispatch configmap trait definition") + cmTrait := &v1beta1.TraitDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmTraitName, + Namespace: "vela-system", + }, + Spec: v1beta1.TraitDefinitionSpec{ + Stage: v1beta1.PostDispatch, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +outputs: statusConfigMap: { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: context.name + "-status" + namespace: context.namespace + } + data: { + replicas: "\(context.output.status.replicas)" + readyReplicas: "\(context.output.status.readyReplicas)" + componentName: context.name + } +} +`, + }, + }, + Status: &common.Status{ + HealthPolicy: `cm: context.outputs.statusConfigMap +_isHealth: cm.data.readyReplicas != "2" +isHealth: *_isHealth | bool +`, + }, + }, + } + Expect(k8sClient.Create(ctx, cmTrait)).Should(Succeed()) + DeferCleanup(func() { + _ = k8sClient.Delete(ctx, deploymentTrait) + _ = k8sClient.Delete(ctx, cmTrait) + }) + + app := &v1beta1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app-with-postdispatch-status", + Namespace: namespace, + }, + Spec: v1beta1.ApplicationSpec{ + Components: []common.ApplicationComponent{ + { + Name: "test-deployment", + Type: "webservice", + Properties: &runtime.RawExtension{Raw: []byte(`{"image":"nginx:1.21","port":80,"cpu":"100m","memory":"128Mi"}`)}, + Traits: []common.ApplicationTrait{ + {Type: "scaler", Properties: &runtime.RawExtension{Raw: []byte(`{"replicas":3}`)}}, + {Type: deploymentTraitName, Properties: &runtime.RawExtension{Raw: []byte(`{"name":"trait-deployment","image":"nginx:1.21"}`)}}, + {Type: cmTraitName}, + }, + }, + }, + }, + } + DeferCleanup(func() { _ = k8sClient.Delete(ctx, app) }) + + By("Creating application that uses PostDispatch traits") + Expect(k8sClient.Create(ctx, app)).Should(Succeed()) + + By("Waiting for application, component, and traits to become healthy") + Eventually(func(g Gomega) { + checkApp := &v1beta1.Application{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "app-with-postdispatch-status"}, checkApp)).Should(Succeed()) + g.Expect(checkApp.Status.Phase).Should(Equal(common.ApplicationRunning)) + g.Expect(checkApp.Status.Services).ShouldNot(BeEmpty()) + for _, svc := range checkApp.Status.Services { + g.Expect(svc.Healthy).Should(BeTrue()) + for _, traitStatus := range svc.Traits { + g.Expect(traitStatus.Healthy).Should(BeTrue()) + } + } + }, 120*time.Second, 3*time.Second).Should(Succeed()) + By("Ensuring the primary component deployment is healthy") + Eventually(func(g Gomega) { + componentDeploy := &appsv1.Deployment{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "test-deployment"}, componentDeploy)).Should(Succeed()) + g.Expect(componentDeploy.Status.ReadyReplicas).Should(Equal(int32(3))) + g.Expect(componentDeploy.Status.Replicas).Should(Equal(int32(3))) + }, 90*time.Second, 3*time.Second).Should(Succeed()) + + By("Ensuring PostDispatch trait-managed deployment reflects component status") + Eventually(func(g Gomega) { + traitDeploy := &appsv1.Deployment{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "trait-deployment"}, traitDeploy)).Should(Succeed()) + g.Expect(traitDeploy.Status.ReadyReplicas).Should(Equal(int32(3))) + g.Expect(traitDeploy.Status.Replicas).Should(Equal(int32(3))) + }, 90*time.Second, 3*time.Second).Should(Succeed()) + + By("Ensuring PostDispatch status ConfigMap reflects healthy state") + Eventually(func(g Gomega) { + statusCM := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "test-deployment-status"}, statusCM)).Should(Succeed()) + g.Expect(statusCM.Data["componentName"]).Should(Equal("test-deployment")) + g.Expect(statusCM.Data["readyReplicas"]).Should(Equal("3")) + g.Expect(statusCM.Data["replicas"]).Should(Equal("3")) + }, 90*time.Second, 3*time.Second).Should(Succeed()) + }) + + It("Should surface unhealthy status when PostDispatch trait deployment crashes at later stage", func() { + deploymentTraitName := "test-deployment-trait-" + randomNamespaceName("") + cmTraitName := "test-cm-trait-" + randomNamespaceName("") + + By("Creating PostDispatch deployment trait definition") + deploymentTrait := &v1beta1.TraitDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentTraitName, + Namespace: "vela-system", + }, + Spec: v1beta1.TraitDefinitionSpec{ + Stage: v1beta1.PostDispatch, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +outputs: statusPod: { + apiVersion: "apps/v1" + kind: "Deployment" + metadata: { + name: parameter.name + } + spec: { + replicas: context.output.status.replicas + selector: matchLabels: { + app: parameter.name + } + template: { + metadata: labels: { + app: parameter.name + } + spec: containers: [{ + name: parameter.name + image: parameter.image + command: ["sh", "-c"] + args: [""" + echo "Starting NGINX..." + nginx -g "daemon off;" & + sleep 80 + echo "Simulating crash now..." + killall nginx + sleep 5 + exit 1 + """] + ports: [{ containerPort: 80 }] + }] + } + } +} + +parameter: { + name: string + image: string +} +`, + }, + }, + Status: &common.Status{ + HealthPolicy: `pod: context.outputs.statusPod +ready: { + updatedReplicas: *0 | int + readyReplicas: *0 | int + replicas: *0 | int + observedGeneration: *0 | int +} & { + if pod.status.updatedReplicas != _|_ { + updatedReplicas: pod.status.updatedReplicas + } + if pod.status.readyReplicas != _|_ { + readyReplicas: pod.status.readyReplicas + } + if pod.status.replicas != _|_ { + replicas: pod.status.replicas + } + if pod.status.observedGeneration != _|_ { + observedGeneration: pod.status.observedGeneration + } +} +_isHealth: (pod.spec.replicas == ready.readyReplicas) && (pod.spec.replicas == ready.updatedReplicas) && (pod.spec.replicas == ready.replicas) && (ready.observedGeneration == pod.metadata.generation || ready.observedGeneration > pod.metadata.generation) +isHealth: *_isHealth | bool +if pod.metadata.annotations != _|_ { + if pod.metadata.annotations["app.oam.dev/disable-health-check"] != _|_ { + isHealth: true + } +} +`, + }, + }, + } + Expect(k8sClient.Create(ctx, deploymentTrait)).Should(Succeed()) + + By("Creating PostDispatch configmap trait definition") + cmTrait := &v1beta1.TraitDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmTraitName, + Namespace: "vela-system", + }, + Spec: v1beta1.TraitDefinitionSpec{ + Stage: v1beta1.PostDispatch, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +outputs: statusConfigMap: { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: context.name + "-status" + namespace: context.namespace + } + data: { + replicas: "\(context.output.status.replicas)" + readyReplicas: "\(context.output.status.readyReplicas)" + componentName: context.name + } +} +`, + }, + }, + Status: &common.Status{ + HealthPolicy: `cm: context.outputs.statusConfigMap +_isHealth: cm.data.readyReplicas != "2" +isHealth: *_isHealth | bool +`, + }, + }, + } + Expect(k8sClient.Create(ctx, cmTrait)).Should(Succeed()) + DeferCleanup(func() { + _ = k8sClient.Delete(ctx, deploymentTrait) + _ = k8sClient.Delete(ctx, cmTrait) + }) + + app := &v1beta1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app-with-postdispatch-status", + Namespace: namespace, + }, + Spec: v1beta1.ApplicationSpec{ + Components: []common.ApplicationComponent{ + { + Name: "test-deployment", + Type: "webservice", + Properties: &runtime.RawExtension{Raw: []byte(`{"image":"nginx:1.21","port":80,"cpu":"100m","memory":"128Mi"}`)}, + Traits: []common.ApplicationTrait{ + {Type: "scaler", Properties: &runtime.RawExtension{Raw: []byte(`{"replicas":3}`)}}, + {Type: deploymentTraitName, Properties: &runtime.RawExtension{Raw: []byte(`{"name":"trait-deployment","image":"nginx:alpine"}`)}}, + {Type: cmTraitName}, + }, + }, + }, + }, + } + DeferCleanup(func() { _ = k8sClient.Delete(ctx, app) }) + + By("Creating application that uses PostDispatch traits") + Expect(k8sClient.Create(ctx, app)).Should(Succeed()) + + By("Waiting for PostDispatch trait to report healthy status") + Eventually(func(g Gomega) { + checkApp := &v1beta1.Application{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp)).Should(Succeed()) + g.Expect(checkApp.Status.Phase).Should(Equal(common.ApplicationRunning)) + g.Expect(checkApp.Status.Services).ShouldNot(BeEmpty()) + svc := checkApp.Status.Services[0] + g.Expect(svc.Healthy).Should(BeTrue()) + foundTrait := false + for _, traitStatus := range svc.Traits { + if traitStatus.Type == deploymentTraitName { + g.Expect(traitStatus.Healthy).Should(BeTrue()) + foundTrait = true + } + } + g.Expect(foundTrait).Should(BeTrue()) + }, 240*time.Second, 5*time.Second).Should(Succeed()) + + By("Waiting for CrashLoopBackOff to flip trait and application unhealthy") + Eventually(func(g Gomega) { + checkApp := &v1beta1.Application{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp)).Should(Succeed()) + g.Expect(checkApp.Status.Services).ShouldNot(BeEmpty()) + svc := checkApp.Status.Services[0] + + traitFound := false + for _, traitStatus := range svc.Traits { + if traitStatus.Type == deploymentTraitName { + traitFound = true + g.Expect(traitStatus.Healthy).Should(BeFalse()) + } + } + g.Expect(traitFound).Should(BeTrue()) + }, 300*time.Second, 5*time.Second).Should(Succeed()) + }) + + It("Should keep PostDispatch trait pending when component image fails", func() { + deploymentTraitName := "test-deployment-trait-" + randomNamespaceName("") + cmTraitName := "test-cm-trait-" + randomNamespaceName("") + + By("Creating PostDispatch deployment trait definition") + deploymentTrait := &v1beta1.TraitDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentTraitName, + Namespace: "vela-system", + }, + Spec: v1beta1.TraitDefinitionSpec{ + Stage: v1beta1.PostDispatch, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +outputs: statusPod: { + apiVersion: "apps/v1" + kind: "Deployment" + metadata: { + name: parameter.name + } + spec: { + replicas: context.output.status.replicas + selector: matchLabels: { + app: parameter.name + } + template: { + metadata: labels: { + app: parameter.name + } + spec: containers: [{ + name: parameter.name + image: parameter.image + }] + } + } +} + +parameter: { + name: string + image: string +} +`, + }, + }, + Status: &common.Status{ + HealthPolicy: `pod: context.outputs.statusPod +ready: { + updatedReplicas: *0 | int + readyReplicas: *0 | int + replicas: *0 | int + observedGeneration: *0 | int +} & { + if pod.status.updatedReplicas != _|_ { + updatedReplicas: pod.status.updatedReplicas + } + if pod.status.readyReplicas != _|_ { + readyReplicas: pod.status.readyReplicas + } + if pod.status.replicas != _|_ { + replicas: pod.status.replicas + } + if pod.status.observedGeneration != _|_ { + observedGeneration: pod.status.observedGeneration + } +} +_isHealth: (pod.spec.replicas == ready.readyReplicas) && (pod.spec.replicas == ready.updatedReplicas) && (pod.spec.replicas == ready.replicas) && (ready.observedGeneration == pod.metadata.generation || ready.observedGeneration > pod.metadata.generation) +isHealth: *_isHealth | bool +if pod.metadata.annotations != _|_ { + if pod.metadata.annotations["app.oam.dev/disable-health-check"] != _|_ { + isHealth: true + } +} +`, + }, + }, + } + Expect(k8sClient.Create(ctx, deploymentTrait)).Should(Succeed()) + + By("Creating PostDispatch configMap trait definition") + cmTrait := &v1beta1.TraitDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmTraitName, + Namespace: "vela-system", + }, + Spec: v1beta1.TraitDefinitionSpec{ + Stage: v1beta1.PostDispatch, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +outputs: statusConfigMap: { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: context.name + "-status" + namespace: context.namespace + } + data: { + replicas: "\(context.output.status.replicas)" + readyReplicas: "\(context.output.status.readyReplicas)" + componentName: context.name + } +} +`, + }, + }, + Status: &common.Status{ + HealthPolicy: `cm: context.outputs.statusConfigMap +_isHealth: cm.data.readyReplicas != "2" +isHealth: *_isHealth | bool +`, + }, + }, + } + Expect(k8sClient.Create(ctx, cmTrait)).Should(Succeed()) + DeferCleanup(func() { + _ = k8sClient.Delete(ctx, deploymentTrait) + _ = k8sClient.Delete(ctx, cmTrait) + }) + + app := &v1beta1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app-with-postdispatch-status", + Namespace: namespace, + }, + Spec: v1beta1.ApplicationSpec{ + Components: []common.ApplicationComponent{ + { + Name: "test-deployment", + Type: "webservice", + Properties: &runtime.RawExtension{Raw: []byte(`{"image":"nginx:1.21abc","port":80,"cpu":"100m","memory":"128Mi"}`)}, + Traits: []common.ApplicationTrait{ + {Type: "scaler", Properties: &runtime.RawExtension{Raw: []byte(`{"replicas":3}`)}}, + {Type: deploymentTraitName, Properties: &runtime.RawExtension{Raw: []byte(`{"name":"trait-deployment","image":"nginx:1.21"}`)}}, + {Type: cmTraitName}, + }, + }, + }, + }, + } + DeferCleanup(func() { _ = k8sClient.Delete(ctx, app) }) + + By("Creating application that uses PostDispatch traits") + Expect(k8sClient.Create(ctx, app)).Should(Succeed()) + + By("Waiting for trait to remain pending and not show in status while component image fails") + Eventually(func(g Gomega) { + checkApp := &v1beta1.Application{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp)).Should(Succeed()) + g.Expect(checkApp.Status.Services).ShouldNot(BeEmpty()) + svc := checkApp.Status.Services[0] + g.Expect(svc.Healthy).Should(BeFalse()) + + traitFound := false + for _, traitStatus := range svc.Traits { + if traitStatus.Type == deploymentTraitName { + traitFound = true + } + } + g.Expect(traitFound).Should(BeFalse()) + }, 180*time.Second, 5*time.Second).Should(Succeed()) + }) + }) + + Context("Test PostDispatch trait with component status", func() { + It("Should render PostDispatch trait after component is healthy and has status", func() { + compDefName := "test-worker-" + randomNamespaceName("") + traitDefName := "test-status-trait-" + randomNamespaceName("") + + By("Creating ComponentDefinition") + compDef := &v1beta1.ComponentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: compDefName, + Namespace: "vela-system", + }, + Spec: v1beta1.ComponentDefinitionSpec{ + Workload: common.WorkloadTypeDescriptor{ + Definition: common.WorkloadGVK{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + }, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +output: { + apiVersion: "apps/v1" + kind: "Deployment" + metadata: { + name: parameter.name + labels: { + app: parameter.name + } + } + spec: { + replicas: parameter.replicas + selector: matchLabels: { + app: parameter.name + } + template: { + metadata: labels: { + app: parameter.name + } + spec: containers: [{ + name: parameter.name + image: parameter.image + }] + } + } +} + +parameter: { + name: string + image: string + replicas: *1 | int +} +`, + }, + }, + Status: &common.Status{ + HealthPolicy: `isHealth: context.output.status.readyReplicas > 0`, + }, + }, + } + Expect(k8sClient.Create(ctx, compDef)).Should(Succeed()) + + By("Creating PostDispatch TraitDefinition") + traitDef := &v1beta1.TraitDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: traitDefName, + Namespace: "vela-system", + }, + Spec: v1beta1.TraitDefinitionSpec{ + Stage: v1beta1.PostDispatch, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +outputs: statusConfigMap: { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: context.name + "-status" + namespace: context.namespace + } + data: { + // Access the component's output status + replicas: "\(context.output.status.replicas)" + readyReplicas: "\(context.output.status.readyReplicas)" + componentName: context.name + } +} +`, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, traitDef)).Should(Succeed()) + + By("Creating Application with PostDispatch trait") + app := &v1beta1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-postdispatch-app", + Namespace: namespace, + }, + Spec: v1beta1.ApplicationSpec{ + Components: []common.ApplicationComponent{ + { + Name: "test-component", + Type: compDefName, + Properties: &runtime.RawExtension{ + Raw: []byte(`{"name":"test-worker","image":"nginx:1.14.2","replicas":1}`), + }, + Traits: []common.ApplicationTrait{ + { + Type: traitDefName, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, app)).Should(Succeed()) + + By("Waiting for Application to be running") + Eventually(func(g Gomega) { + checkApp := &v1beta1.Application{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "test-postdispatch-app"}, checkApp)).Should(Succeed()) + g.Expect(checkApp.Status.Phase).Should(Equal(common.ApplicationRunning)) + }, 60*time.Second, 3*time.Second).Should(Succeed()) + + By("Verifying component Deployment is created and healthy") + Eventually(func(g Gomega) { + deploy := &unstructured.Unstructured{} + deploy.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }) + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "test-worker"}, deploy)).Should(Succeed()) + + status, found, _ := unstructured.NestedMap(deploy.Object, "status") + g.Expect(found).Should(BeTrue()) + g.Expect(status).ShouldNot(BeNil()) + + replicas, _, _ := unstructured.NestedInt64(status, "replicas") + g.Expect(replicas).Should(Equal(int64(1))) + }, 30*time.Second, 2*time.Second).Should(Succeed()) + + By("Verifying PostDispatch trait ConfigMap was created with status data") + Eventually(func(g Gomega) { + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "test-component-status"}, cm)).Should(Succeed()) + g.Expect(cm.Data).ShouldNot(BeNil()) + g.Expect(cm.Data["componentName"]).Should(Equal("test-component")) + g.Expect(cm.Data["replicas"]).Should(Equal("1")) + g.Expect(cm.Data["readyReplicas"]).Should(Equal("1")) + }, 300*time.Second, 3*time.Second).Should(Succeed()) + + By("Verifying PostDispatch trait appears in application status") + Eventually(func(g Gomega) { + checkApp := &v1beta1.Application{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "test-postdispatch-app"}, checkApp)).Should(Succeed()) + g.Expect(checkApp.Status.Services).Should(HaveLen(1)) + + svc := checkApp.Status.Services[0] + g.Expect(svc.Healthy).Should(BeTrue()) + + // Find the PostDispatch trait in the status + var foundTrait bool + for _, trait := range svc.Traits { + if trait.Type == traitDefName { + foundTrait = true + g.Expect(trait.Healthy).Should(BeTrue()) + break + } + } + g.Expect(foundTrait).Should(BeTrue(), "PostDispatch trait should appear in application status") + }, 30*time.Second, 2*time.Second).Should(Succeed()) + + By("Cleaning up test resources") + Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, traitDef)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, compDef)).Should(Succeed()) + }) + + It("Should show PostDispatch trait as pending before component is healthy", func() { + compDefName := "test-slow-worker-" + randomNamespaceName("") + traitDefName := "test-pending-trait-" + randomNamespaceName("") + + By("Creating ComponentDefinition with readiness probe") + compDef := &v1beta1.ComponentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: compDefName, + Namespace: "vela-system", + }, + Spec: v1beta1.ComponentDefinitionSpec{ + Workload: common.WorkloadTypeDescriptor{ + Definition: common.WorkloadGVK{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + }, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +output: { + apiVersion: "apps/v1" + kind: "Deployment" + metadata: { + name: parameter.name + } + spec: { + replicas: parameter.replicas + selector: matchLabels: { + app: parameter.name + } + template: { + metadata: labels: { + app: parameter.name + } + spec: containers: [{ + name: parameter.name + image: parameter.image + // Add readiness probe that delays health + readinessProbe: { + httpGet: { + path: "/" + port: 80 + } + initialDelaySeconds: 10 + periodSeconds: 2 + } + }] + } + } +} + +parameter: { + name: string + image: string + replicas: *1 | int +} +`, + }, + }, + Status: &common.Status{ + HealthPolicy: `isHealth: context.output.status.readyReplicas > 0`, + }, + }, + } + Expect(k8sClient.Create(ctx, compDef)).Should(Succeed()) + + By("Creating PostDispatch TraitDefinition") + traitDef := &v1beta1.TraitDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: traitDefName, + Namespace: "vela-system", + }, + Spec: v1beta1.TraitDefinitionSpec{ + Stage: v1beta1.PostDispatch, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +outputs: marker: { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: context.name + "-marker" + namespace: context.namespace + } + data: { + status: "deployed" + } +} +`, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, traitDef)).Should(Succeed()) + + By("Creating Application") + app := &v1beta1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pending-app", + Namespace: namespace, + }, + Spec: v1beta1.ApplicationSpec{ + Components: []common.ApplicationComponent{ + { + Name: "slow-component", + Type: compDefName, + Properties: &runtime.RawExtension{ + Raw: []byte(`{"name":"slow-worker","image":"nginx:1.14.2","replicas":1}`), + }, + Traits: []common.ApplicationTrait{ + { + Type: traitDefName, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, app)).Should(Succeed()) + + By("Verifying PostDispatch trait shows as pending while component is not healthy") + Eventually(func(g Gomega) { + checkApp := &v1beta1.Application{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "test-pending-app"}, checkApp)).Should(Succeed()) + + // Application should be running workflow while component is deploying + g.Expect(checkApp.Status.Phase).Should(BeElementOf(common.ApplicationRunningWorkflow, common.ApplicationRunning)) + + // Services must be populated to check trait status + g.Expect(checkApp.Status.Services).ShouldNot(BeEmpty(), "Expected Services to be populated in application status") + + svc := checkApp.Status.Services[0] + foundPendingTrait := false + for _, trait := range svc.Traits { + if trait.Type == traitDefName { + // Trait should show as pending and not healthy + foundPendingTrait = true + break + } + } + // If workflow is running, we will not be able to see the pending trait status yet + if checkApp.Status.Phase == common.ApplicationRunningWorkflow { + g.Expect(foundPendingTrait).Should(BeFalse()) + } + }, 20*time.Second, 500*time.Millisecond).Should(Succeed()) + + By("Waiting for component to become healthy and PostDispatch trait to be deployed") + Eventually(func(g Gomega) { + // Check if the PostDispatch ConfigMap has been created + cm := &corev1.ConfigMap{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "slow-component-marker"}, cm)).Should(Succeed()) + g.Expect(cm.Data).ShouldNot(BeNil()) + g.Expect(cm.Data["status"]).Should(Equal("deployed")) + }, 90*time.Second, 3*time.Second).Should(Succeed()) + + By("Verifying PostDispatch trait is no longer pending") + Eventually(func(g Gomega) { + checkApp := &v1beta1.Application{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "test-pending-app"}, checkApp)).Should(Succeed()) + + // Services must be populated + g.Expect(checkApp.Status.Services).ShouldNot(BeEmpty(), "Expected Services to be populated in application status") + + svc := checkApp.Status.Services[0] + foundTrait := false + for _, trait := range svc.Traits { + if trait.Type == traitDefName { + foundTrait = true + // Trait should be healthy, not pending, and not waiting anymore + g.Expect(trait.Healthy).Should(BeTrue()) + break + } + } + // The trait entry must exist in the status + g.Expect(foundTrait).Should(BeTrue(), "Expected to find trait in application status") + }, 30*time.Second, 2*time.Second).Should(Succeed()) + + By("Cleaning up") + Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, traitDef)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, compDef)).Should(Succeed()) + }) + + It("Should fail when PostDispatch trait accesses status without health policy", func() { + compDefName := "test-no-health-" + randomNamespaceName("") + traitDefName := "test-status-access-trait-" + randomNamespaceName("") + + By("Creating ComponentDefinition WITHOUT health policy") + compDef := &v1beta1.ComponentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: compDefName, + Namespace: "vela-system", + }, + Spec: v1beta1.ComponentDefinitionSpec{ + Workload: common.WorkloadTypeDescriptor{ + Definition: common.WorkloadGVK{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + }, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +output: { + apiVersion: "apps/v1" + kind: "Deployment" + metadata: { + name: parameter.name + } + spec: { + replicas: parameter.replicas + selector: matchLabels: { + app: parameter.name + } + template: { + metadata: labels: { + app: parameter.name + } + spec: containers: [{ + name: parameter.name + image: parameter.image + }] + } + } +} + +parameter: { + name: string + image: string + replicas: *1 | int +} +`, + }, + }, + // Deliberately NO Status or HealthPolicy - component will be immediately healthy + }, + } + Expect(k8sClient.Create(ctx, compDef)).Should(Succeed()) + + By("Creating PostDispatch trait that accesses output.status") + traitDef := &v1beta1.TraitDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: traitDefName, + Namespace: "vela-system", + }, + Spec: v1beta1.TraitDefinitionSpec{ + Stage: v1beta1.PostDispatch, + Schematic: &common.Schematic{ + CUE: &common.CUE{ + Template: ` +outputs: statusConfigMap: { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: context.name + "-status" + namespace: context.namespace + } + data: { + // This will fail because status fields don't exist + replicas: "\(context.output.status.replicas)" + readyReplicas: "\(context.output.status.readyReplicas)" + componentName: context.name + } +} +`, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, traitDef)).Should(Succeed()) + + By("Creating Application with PostDispatch trait") + app := &v1beta1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-no-health-app", + Namespace: namespace, + }, + Spec: v1beta1.ApplicationSpec{ + Components: []common.ApplicationComponent{ + { + Name: "test-component", + Type: compDefName, + Properties: &runtime.RawExtension{ + Raw: []byte(`{"name":"test-worker","image":"nginx:1.14.2","replicas":1}`), + }, + Traits: []common.ApplicationTrait{ + { + Type: traitDefName, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, app)).Should(Succeed()) + + By("Verifying Application enters failed state due to rendering error") + Eventually(func(g Gomega) { + checkApp := &v1beta1.Application{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "test-no-health-app"}, checkApp)).Should(Succeed()) + + // Application should fail during workflow execution + if checkApp.Status.Workflow != nil { + // Workflow should either fail or a step should fail + workflowFailed := string(checkApp.Status.Workflow.Phase) == "failed" + stepFailed := false + for _, step := range checkApp.Status.Workflow.Steps { + if string(step.Phase) == "failed" { + stepFailed = true + // Should have error message about CUE evaluation or status access + g.Expect(step.Message).Should(Or( + ContainSubstring("failed to evaluate"), + ContainSubstring("failed to render"), + ContainSubstring("PostDispatch"), + ContainSubstring("status"), + )) + break + } + } + g.Expect(workflowFailed || stepFailed).Should(BeTrue(), "Expected workflow or step to fail due to status access error") + } + }, 30*time.Second, 2*time.Second).Should(Succeed()) + + By("Cleaning up") + Expect(k8sClient.Delete(ctx, app)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, traitDef)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, compDef)).Should(Succeed()) + }) + }) +})