mirror of
https://github.com/kubevela/kubevela.git
synced 2026-08-18 20:17:04 +00:00
Feat: add if in workflow (#3941)
* Feat: add if in workflow struct Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Feat: implement the if in workflow Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Feat: support dependency and skip for suspend step Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Fix: fix the rebase from sub steps Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Fix: fix the lint Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Feat: support if in sub steps Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Feat: add tests in application controller Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Fix: fix the lint Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Test: add more tests in discover and custom Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Lint: fix lint Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Tests: add more tests in application controller Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Fix: change failed after retries into reason Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * Fix: fix the terminate cli Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * fix lint Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * remove the terminate workflow to pkg and add feature gates Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * resolve comments Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * nit fix Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com> * make finish condition more clear Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>
This commit is contained in:
@@ -347,6 +347,8 @@ type WorkflowStep struct {
|
||||
|
||||
SubSteps []WorkflowSubStep `json:"subSteps,omitempty"`
|
||||
|
||||
If string `json:"if,omitempty"`
|
||||
|
||||
DependsOn []string `json:"dependsOn,omitempty"`
|
||||
|
||||
Inputs StepInputs `json:"inputs,omitempty"`
|
||||
@@ -364,6 +366,8 @@ type WorkflowSubStep struct {
|
||||
// +kubebuilder:pruning:PreserveUnknownFields
|
||||
Properties *runtime.RawExtension `json:"properties,omitempty"`
|
||||
|
||||
If string `json:"if,omitempty"`
|
||||
|
||||
DependsOn []string `json:"dependsOn,omitempty"`
|
||||
|
||||
Inputs StepInputs `json:"inputs,omitempty"`
|
||||
@@ -397,6 +401,8 @@ const (
|
||||
WorkflowStepPhaseSucceeded WorkflowStepPhase = "succeeded"
|
||||
// WorkflowStepPhaseFailed will report error in `message`.
|
||||
WorkflowStepPhaseFailed WorkflowStepPhase = "failed"
|
||||
// WorkflowStepPhaseSkipped will make the controller skip the step.
|
||||
WorkflowStepPhaseSkipped WorkflowStepPhase = "skipped"
|
||||
// WorkflowStepPhaseStopped will make the controller stop the workflow.
|
||||
WorkflowStepPhaseStopped WorkflowStepPhase = "stopped"
|
||||
// WorkflowStepPhaseRunning will make the controller continue the workflow.
|
||||
|
||||
@@ -53,11 +53,12 @@ helm install --create-namespace -n vela-system kubevela kubevela/vela-core --wai
|
||||
|
||||
### KubeVela workflow parameters
|
||||
|
||||
| Name | Description | Value |
|
||||
| -------------------------------------- | ------------------------------------------------------ | ----- |
|
||||
| `workflow.backoff.maxTime.waitState` | The max backoff time of workflow in a wait condition | `60` |
|
||||
| `workflow.backoff.maxTime.failedState` | The max backoff time of workflow in a failed condition | `300` |
|
||||
| `workflow.step.errorRetryTimes` | The max retry times of a failed workflow step | `10` |
|
||||
| Name | Description | Value |
|
||||
| -------------------------------------- | ------------------------------------------------------ | ------- |
|
||||
| `workflow.enableSuspendOnFailure` | Enable suspend on workflow failure | `false` |
|
||||
| `workflow.backoff.maxTime.waitState` | The max backoff time of workflow in a wait condition | `60` |
|
||||
| `workflow.backoff.maxTime.failedState` | The max backoff time of workflow in a failed condition | `300` |
|
||||
| `workflow.step.errorRetryTimes` | The max retry times of a failed workflow step | `10` |
|
||||
|
||||
|
||||
### KubeVela controller parameters
|
||||
|
||||
@@ -2209,6 +2209,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of
|
||||
WorkflowStep
|
||||
@@ -2253,6 +2255,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input
|
||||
of WorkflowStep
|
||||
@@ -3954,6 +3958,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -3995,6 +4001,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of
|
||||
WorkflowStep
|
||||
|
||||
@@ -1020,6 +1020,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -1061,6 +1063,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of
|
||||
WorkflowStep
|
||||
|
||||
@@ -42,6 +42,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -83,6 +85,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -147,6 +151,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -188,6 +194,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
|
||||
@@ -172,6 +172,7 @@ spec:
|
||||
- "--max-workflow-wait-backoff-time={{ .Values.workflow.backoff.maxTime.waitState }}"
|
||||
- "--max-workflow-failed-backoff-time={{ .Values.workflow.backoff.maxTime.failedState }}"
|
||||
- "--max-workflow-step-error-retry-times={{ .Values.workflow.step.errorRetryTimes }}"
|
||||
- "--feature-gates=EnableSuspendOnFailure={{- .Values.workflow.enableSuspendOnFailure | toString -}}"
|
||||
- "--feature-gates=AuthenticateApplication={{- .Values.authentication.enabled | toString -}}"
|
||||
{{ if .Values.authentication.enabled }}
|
||||
{{ if .Values.authentication.withUser }}
|
||||
|
||||
@@ -35,10 +35,12 @@ dependCheckWait: 30s
|
||||
|
||||
## @section KubeVela workflow parameters
|
||||
|
||||
## @param workflow.enableSuspendOnFailure Enable suspend on workflow failure
|
||||
## @param workflow.backoff.maxTime.waitState The max backoff time of workflow in a wait condition
|
||||
## @param workflow.backoff.maxTime.failedState The max backoff time of workflow in a failed condition
|
||||
## @param workflow.step.errorRetryTimes The max retry times of a failed workflow step
|
||||
workflow:
|
||||
enableSuspendOnFailure: false
|
||||
backoff:
|
||||
maxTime:
|
||||
waitState: 60
|
||||
|
||||
@@ -72,11 +72,12 @@ helm install --create-namespace -n vela-system kubevela kubevela/vela-minimal --
|
||||
|
||||
### KubeVela workflow parameters
|
||||
|
||||
| Name | Description | Value |
|
||||
| -------------------------------------- | ------------------------------------------------------ | ----- |
|
||||
| `workflow.backoff.maxTime.waitState` | The max backoff time of workflow in a wait condition | `60` |
|
||||
| `workflow.backoff.maxTime.failedState` | The max backoff time of workflow in a failed condition | `300` |
|
||||
| `workflow.step.errorRetryTimes` | The max retry times of a failed workflow step | `10` |
|
||||
| Name | Description | Value |
|
||||
| -------------------------------------- | ------------------------------------------------------ | ------- |
|
||||
| `workflow.enableSuspendOnFailure` | Enable suspend on workflow failure | `false` |
|
||||
| `workflow.backoff.maxTime.waitState` | The max backoff time of workflow in a wait condition | `60` |
|
||||
| `workflow.backoff.maxTime.failedState` | The max backoff time of workflow in a failed condition | `300` |
|
||||
| `workflow.step.errorRetryTimes` | The max retry times of a failed workflow step | `10` |
|
||||
|
||||
|
||||
### KubeVela controller parameters
|
||||
|
||||
@@ -2209,6 +2209,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of
|
||||
WorkflowStep
|
||||
@@ -2253,6 +2255,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input
|
||||
of WorkflowStep
|
||||
@@ -3954,6 +3958,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -3995,6 +4001,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of
|
||||
WorkflowStep
|
||||
|
||||
@@ -1020,6 +1020,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -1061,6 +1063,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of
|
||||
WorkflowStep
|
||||
|
||||
@@ -142,6 +142,7 @@ spec:
|
||||
- "--max-workflow-wait-backoff-time={{ .Values.workflow.backoff.maxTime.waitState }}"
|
||||
- "--max-workflow-failed-backoff-time={{ .Values.workflow.backoff.maxTime.failedState }}"
|
||||
- "--max-workflow-step-error-retry-times={{ .Values.workflow.step.errorRetryTimes }}"
|
||||
- "--feature-gates=EnableSuspendOnFailure={{- .Values.workflow.enableSuspendOnFailure | toString -}}"
|
||||
- "--feature-gates=AuthenticateApplication={{- .Values.authentication.enabled | toString -}}"
|
||||
{{ if .Values.authentication.enabled }}
|
||||
{{ if .Values.authentication.withUser }}
|
||||
|
||||
@@ -38,10 +38,12 @@ dependCheckWait: 30s
|
||||
|
||||
## @section KubeVela workflow parameters
|
||||
|
||||
## @param workflow.enableSuspendOnFailure Enable suspend on workflow failure
|
||||
## @param workflow.backoff.maxTime.waitState The max backoff time of workflow in a wait condition
|
||||
## @param workflow.backoff.maxTime.failedState The max backoff time of workflow in a failed condition
|
||||
## @param workflow.step.errorRetryTimes The max retry times of a failed workflow step
|
||||
workflow:
|
||||
enableSuspendOnFailure: false
|
||||
backoff:
|
||||
maxTime:
|
||||
waitState: 60
|
||||
|
||||
@@ -2209,6 +2209,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of
|
||||
WorkflowStep
|
||||
@@ -2253,6 +2255,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input
|
||||
of WorkflowStep
|
||||
@@ -3954,6 +3958,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -3995,6 +4001,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of
|
||||
WorkflowStep
|
||||
|
||||
@@ -1021,6 +1021,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -1062,6 +1064,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of
|
||||
WorkflowStep
|
||||
|
||||
@@ -42,6 +42,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -83,6 +85,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -147,6 +151,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
@@ -188,6 +194,8 @@ spec:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
if:
|
||||
type: string
|
||||
inputs:
|
||||
description: StepInputs defines variable input of WorkflowStep
|
||||
items:
|
||||
|
||||
@@ -43,6 +43,7 @@ import (
|
||||
"github.com/oam-dev/kubevela/pkg/oam/util"
|
||||
utils2 "github.com/oam-dev/kubevela/pkg/utils"
|
||||
"github.com/oam-dev/kubevela/pkg/utils/apply"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/tasks/custom"
|
||||
)
|
||||
|
||||
// WorkflowService workflow manage api
|
||||
@@ -577,8 +578,7 @@ func (w *workflowServiceImpl) TerminateRecord(ctx context.Context, appModel *mod
|
||||
return err
|
||||
}
|
||||
|
||||
oamApp.Status.Workflow.Terminated = true
|
||||
if err := w.KubeClient.Status().Patch(ctx, oamApp, client.Merge); err != nil {
|
||||
if err := TerminateWorkflow(ctx, w.KubeClient, oamApp); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.syncWorkflowStatus(ctx, oamApp, recordName, oamApp.Name); err != nil {
|
||||
@@ -588,6 +588,42 @@ func (w *workflowServiceImpl) TerminateRecord(ctx context.Context, appModel *mod
|
||||
return nil
|
||||
}
|
||||
|
||||
// TerminateWorkflow terminate workflow
|
||||
func TerminateWorkflow(ctx context.Context, kubecli client.Client, app *v1beta1.Application) error {
|
||||
// set the workflow terminated to true
|
||||
app.Status.Workflow.Terminated = true
|
||||
steps := app.Status.Workflow.Steps
|
||||
for i, step := range steps {
|
||||
switch step.Phase {
|
||||
case common.WorkflowStepPhaseFailed:
|
||||
if step.Reason != custom.StatusReasonFailedAfterRetries {
|
||||
steps[i].Reason = custom.StatusReasonTerminate
|
||||
}
|
||||
case common.WorkflowStepPhaseRunning:
|
||||
steps[i].Phase = common.WorkflowStepPhaseFailed
|
||||
steps[i].Reason = custom.StatusReasonTerminate
|
||||
default:
|
||||
}
|
||||
for j, sub := range step.SubStepsStatus {
|
||||
switch sub.Phase {
|
||||
case common.WorkflowStepPhaseFailed:
|
||||
if sub.Reason != custom.StatusReasonFailedAfterRetries {
|
||||
steps[i].SubStepsStatus[j].Phase = custom.StatusReasonTerminate
|
||||
}
|
||||
case common.WorkflowStepPhaseRunning:
|
||||
steps[i].SubStepsStatus[j].Phase = common.WorkflowStepPhaseFailed
|
||||
steps[i].SubStepsStatus[j].Reason = custom.StatusReasonTerminate
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := kubecli.Status().Patch(ctx, app, client.Merge); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *workflowServiceImpl) RollbackRecord(ctx context.Context, appModel *model.Application, workflow *model.Workflow, recordName, revisionVersion string) error {
|
||||
if revisionVersion == "" {
|
||||
// find the latest complete revision version
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
@@ -47,6 +48,7 @@ import (
|
||||
common2 "github.com/oam-dev/kubevela/pkg/controller/common"
|
||||
core "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/packages"
|
||||
"github.com/oam-dev/kubevela/pkg/features"
|
||||
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
|
||||
"github.com/oam-dev/kubevela/pkg/monitor/metrics"
|
||||
"github.com/oam-dev/kubevela/pkg/oam"
|
||||
@@ -229,7 +231,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
|
||||
handler.app.Status.Workflow.SuspendState = ""
|
||||
return r.gcResourceTrackers(logCtx, handler, common.ApplicationRunningWorkflow, false, false)
|
||||
}
|
||||
if !workflow.IsFailedAfterRetry(app) {
|
||||
if !workflow.IsFailedAfterRetry(app) || !feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
|
||||
r.stateKeep(logCtx, handler, app)
|
||||
}
|
||||
return r.gcResourceTrackers(logCtx, handler, common.ApplicationWorkflowSuspending, false, true)
|
||||
|
||||
@@ -23,8 +23,11 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
@@ -50,6 +53,7 @@ import (
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
|
||||
stdv1alpha1 "github.com/oam-dev/kubevela/apis/standard.oam.dev/v1alpha1"
|
||||
velatypes "github.com/oam-dev/kubevela/apis/types"
|
||||
"github.com/oam-dev/kubevela/pkg/features"
|
||||
"github.com/oam-dev/kubevela/pkg/oam"
|
||||
"github.com/oam-dev/kubevela/pkg/oam/testutil"
|
||||
"github.com/oam-dev/kubevela/pkg/oam/util"
|
||||
@@ -1896,6 +1900,7 @@ var _ = Describe("Test Application Controller", func() {
|
||||
})
|
||||
|
||||
It("application with dag workflow failed after retries", func() {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
|
||||
ns := corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dag-failed-after-retries",
|
||||
@@ -1938,7 +1943,7 @@ var _ = Describe("Test Application Controller", func() {
|
||||
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunningWorkflow))
|
||||
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageInitializingWorkflow))
|
||||
|
||||
By("verify the first twenty reconciles")
|
||||
By("verify the first ten reconciles")
|
||||
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes; i++ {
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
@@ -1947,12 +1952,13 @@ var _ = Describe("Test Application Controller", func() {
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
|
||||
}
|
||||
|
||||
By("application should be suspended after failed twenty reconciles")
|
||||
By("application should be suspended after failed max reconciles")
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationWorkflowSuspending))
|
||||
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageFailedAfterRetries))
|
||||
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageSuspendFailedAfterRetries))
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Reason).Should(BeEquivalentTo(custom.StatusReasonFailedAfterRetries))
|
||||
|
||||
By("resume the suspended application")
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
@@ -1984,7 +1990,7 @@ var _ = Describe("Test Application Controller", func() {
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunningWorkflow))
|
||||
|
||||
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes+1; i++ {
|
||||
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes-1; i++ {
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunningWorkflow))
|
||||
@@ -1992,9 +1998,18 @@ var _ = Describe("Test Application Controller", func() {
|
||||
Expect(checkApp.Status.Workflow.Steps[0].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseRunning))
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
|
||||
}
|
||||
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunningWorkflow))
|
||||
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(string(common.WorkflowStateExecuting)))
|
||||
Expect(checkApp.Status.Workflow.Steps[0].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseRunning))
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Reason).Should(BeEquivalentTo(custom.StatusReasonFailedAfterRetries))
|
||||
})
|
||||
|
||||
It("application with step by step workflow failed after retries", func() {
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
|
||||
ns := corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "step-by-step-failed-after-retries",
|
||||
@@ -2060,12 +2075,13 @@ var _ = Describe("Test Application Controller", func() {
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
|
||||
}
|
||||
|
||||
By("application should be suspended after failed twenty reconciles")
|
||||
By("application should be suspended after failed max reconciles")
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationWorkflowSuspending))
|
||||
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageFailedAfterRetries))
|
||||
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageSuspendFailedAfterRetries))
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Reason).Should(BeEquivalentTo(custom.StatusReasonFailedAfterRetries))
|
||||
|
||||
By("resume the suspended application")
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
@@ -2078,6 +2094,352 @@ var _ = Describe("Test Application Controller", func() {
|
||||
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
|
||||
})
|
||||
|
||||
It("application with sub steps", func() {
|
||||
ns := corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "app-with-sub-steps",
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, &ns)).Should(BeNil())
|
||||
healthComponentDef := &v1beta1.ComponentDefinition{}
|
||||
hCDefJson, _ := yaml.YAMLToJSON([]byte(cdDefWithHealthStatusYaml))
|
||||
Expect(json.Unmarshal(hCDefJson, healthComponentDef)).Should(BeNil())
|
||||
healthComponentDef.Name = "worker-with-health"
|
||||
healthComponentDef.Namespace = "app-with-sub-steps"
|
||||
Expect(k8sClient.Create(ctx, healthComponentDef)).Should(BeNil())
|
||||
app := &v1beta1.Application{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Application",
|
||||
APIVersion: "core.oam.dev/v1beta1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "app-with-sub-steps",
|
||||
Namespace: "app-with-sub-steps",
|
||||
},
|
||||
Spec: v1beta1.ApplicationSpec{
|
||||
Components: []common.ApplicationComponent{
|
||||
{
|
||||
Name: "myweb1",
|
||||
Type: "worker-with-health",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb2",
|
||||
Type: "worker",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb3",
|
||||
Type: "worker-with-health",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
|
||||
},
|
||||
},
|
||||
Workflow: &v1beta1.Workflow{
|
||||
Steps: []v1beta1.WorkflowStep{
|
||||
{
|
||||
Name: "myweb1",
|
||||
Type: "apply-component",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb1"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb2",
|
||||
Type: "step-group",
|
||||
SubSteps: []common.WorkflowSubStep{
|
||||
{
|
||||
Name: "myweb2-sub1",
|
||||
Type: "apply-component",
|
||||
DependsOn: []string{"myweb2-sub2"},
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb2"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb2-sub2",
|
||||
Type: "apply-component",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb3"}`)},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
Expect(k8sClient.Create(context.Background(), app)).Should(BeNil())
|
||||
appKey := types.NamespacedName{Namespace: ns.Name, Name: app.Name}
|
||||
testutil.ReconcileOnceAfterFinalizer(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
expDeployment := &v1.Deployment{}
|
||||
web2Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb2"}
|
||||
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
web3Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb3"}
|
||||
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
|
||||
checkApp := &v1beta1.Application{}
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
web1Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb1"}
|
||||
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(BeNil())
|
||||
|
||||
expDeployment.Status.Replicas = 1
|
||||
expDeployment.Status.ReadyReplicas = 1
|
||||
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
|
||||
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(BeNil())
|
||||
expDeployment.Status.Replicas = 1
|
||||
expDeployment.Status.ReadyReplicas = 1
|
||||
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
|
||||
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(BeNil())
|
||||
expDeployment.Status.Replicas = 1
|
||||
expDeployment.Status.ReadyReplicas = 1
|
||||
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
|
||||
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
checkApp = &v1beta1.Application{}
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunning))
|
||||
})
|
||||
|
||||
It("application with if always in workflow", func() {
|
||||
ns := corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "app-with-if-always-workflow",
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, &ns)).Should(BeNil())
|
||||
healthComponentDef := &v1beta1.ComponentDefinition{}
|
||||
hCDefJson, _ := yaml.YAMLToJSON([]byte(cdDefWithHealthStatusYaml))
|
||||
Expect(json.Unmarshal(hCDefJson, healthComponentDef)).Should(BeNil())
|
||||
healthComponentDef.Name = "worker-with-health"
|
||||
healthComponentDef.Namespace = "app-with-if-always-workflow"
|
||||
Expect(k8sClient.Create(ctx, healthComponentDef)).Should(BeNil())
|
||||
app := &v1beta1.Application{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Application",
|
||||
APIVersion: "core.oam.dev/v1beta1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "app-with-if-always-workflow",
|
||||
Namespace: "app-with-if-always-workflow",
|
||||
},
|
||||
Spec: v1beta1.ApplicationSpec{
|
||||
Components: []common.ApplicationComponent{
|
||||
{
|
||||
Name: "myweb1",
|
||||
Type: "worker-with-health",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb2",
|
||||
Type: "worker",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
|
||||
},
|
||||
{
|
||||
Name: "failed-step",
|
||||
Type: "k8s-objects",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"objects":[{"apiVersion":"v1","kind":"invalid","metadata":{"name":"test1"}}]}`)},
|
||||
},
|
||||
},
|
||||
Workflow: &v1beta1.Workflow{
|
||||
Steps: []v1beta1.WorkflowStep{
|
||||
{
|
||||
Name: "failed-step",
|
||||
Type: "apply-component",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"failed-step"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb1",
|
||||
Type: "apply-component",
|
||||
If: "always",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb1"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb2",
|
||||
Type: "apply-component",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb2"}`)},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
Expect(k8sClient.Create(context.Background(), app)).Should(BeNil())
|
||||
appKey := types.NamespacedName{Namespace: ns.Name, Name: app.Name}
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
By("verify the first ten reconciles")
|
||||
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes; i++ {
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
}
|
||||
|
||||
expDeployment := &v1.Deployment{}
|
||||
web1Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb1"}
|
||||
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
web2Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb2"}
|
||||
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(BeNil())
|
||||
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
|
||||
expDeployment.Status.Replicas = 1
|
||||
expDeployment.Status.ReadyReplicas = 1
|
||||
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
|
||||
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
|
||||
checkApp := &v1beta1.Application{}
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationWorkflowTerminated))
|
||||
})
|
||||
|
||||
It("application with if always in workflow sub steps", func() {
|
||||
ns := corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "app-with-if-always-workflow-sub-steps",
|
||||
},
|
||||
}
|
||||
Expect(k8sClient.Create(ctx, &ns)).Should(BeNil())
|
||||
healthComponentDef := &v1beta1.ComponentDefinition{}
|
||||
hCDefJson, _ := yaml.YAMLToJSON([]byte(cdDefWithHealthStatusYaml))
|
||||
Expect(json.Unmarshal(hCDefJson, healthComponentDef)).Should(BeNil())
|
||||
healthComponentDef.Name = "worker-with-health"
|
||||
healthComponentDef.Namespace = "app-with-if-always-workflow-sub-steps"
|
||||
Expect(k8sClient.Create(ctx, healthComponentDef)).Should(BeNil())
|
||||
app := &v1beta1.Application{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
Kind: "Application",
|
||||
APIVersion: "core.oam.dev/v1beta1",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "app-with-if-always-workflow-sub-steps",
|
||||
Namespace: "app-with-if-always-workflow-sub-steps",
|
||||
},
|
||||
Spec: v1beta1.ApplicationSpec{
|
||||
Components: []common.ApplicationComponent{
|
||||
{
|
||||
Name: "myweb1",
|
||||
Type: "worker-with-health",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb2",
|
||||
Type: "worker-with-health",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb3",
|
||||
Type: "worker",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
|
||||
},
|
||||
{
|
||||
Name: "failed-step",
|
||||
Type: "k8s-objects",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"objects":[{"apiVersion":"v1","kind":"invalid","metadata":{"name":"test1"}}]}`)},
|
||||
},
|
||||
},
|
||||
Workflow: &v1beta1.Workflow{
|
||||
Steps: []v1beta1.WorkflowStep{
|
||||
{
|
||||
Name: "myweb1",
|
||||
Type: "step-group",
|
||||
SubSteps: []common.WorkflowSubStep{
|
||||
{
|
||||
Name: "myweb1-sub1",
|
||||
Type: "apply-component",
|
||||
If: "always",
|
||||
DependsOn: []string{"myweb1-sub2"},
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb1"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb1-sub2",
|
||||
Type: "apply-component",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"failed-step"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb1-sub3",
|
||||
Type: "apply-component",
|
||||
DependsOn: []string{"myweb1-sub1"},
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb2"}`)},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "myweb2",
|
||||
Type: "apply-component",
|
||||
If: "always",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb2"}`)},
|
||||
},
|
||||
{
|
||||
Name: "myweb3",
|
||||
Type: "apply-component",
|
||||
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb3"}`)},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
Expect(k8sClient.Create(context.Background(), app)).Should(BeNil())
|
||||
appKey := types.NamespacedName{Namespace: ns.Name, Name: app.Name}
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
By("verify the first ten reconciles")
|
||||
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes; i++ {
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
}
|
||||
|
||||
expDeployment := &v1.Deployment{}
|
||||
web1Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb1"}
|
||||
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
web2Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb2"}
|
||||
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
web3Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb3"}
|
||||
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(BeNil())
|
||||
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
|
||||
expDeployment.Status.Replicas = 1
|
||||
expDeployment.Status.ReadyReplicas = 1
|
||||
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
|
||||
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(BeNil())
|
||||
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(util.NotFoundMatcher{})
|
||||
expDeployment.Status.Replicas = 1
|
||||
expDeployment.Status.ReadyReplicas = 1
|
||||
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
|
||||
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
|
||||
|
||||
checkApp := &v1beta1.Application{}
|
||||
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
|
||||
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationWorkflowTerminated))
|
||||
})
|
||||
|
||||
It("application with input/output run as dag workflow", func() {
|
||||
ns := corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
|
||||
@@ -122,6 +122,7 @@ func generateStep(ctx context.Context,
|
||||
DependsOn: subStep.DependsOn,
|
||||
Inputs: subStep.Inputs,
|
||||
Outputs: subStep.Outputs,
|
||||
If: subStep.If,
|
||||
}
|
||||
subTask, err := generateStep(ctx, app, workflowStep, taskDiscover, step.Name)
|
||||
if err != nil {
|
||||
|
||||
@@ -33,6 +33,8 @@ const (
|
||||
DeprecatedObjectLabelSelector featuregate.Feature = "DeprecatedObjectLabelSelector"
|
||||
// LegacyResourceTrackerGC enable the gc of legacy resource tracker in managed clusters
|
||||
LegacyResourceTrackerGC featuregate.Feature = "LegacyResourceTrackerGC"
|
||||
// EnableSuspendOnFailure enable suspend on workflow failure
|
||||
EnableSuspendOnFailure featuregate.Feature = "EnableSuspendOnFailure"
|
||||
|
||||
// Edge Features
|
||||
|
||||
@@ -45,6 +47,7 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
|
||||
LegacyObjectTypeIdentifier: {Default: false, PreRelease: featuregate.Alpha},
|
||||
DeprecatedObjectLabelSelector: {Default: false, PreRelease: featuregate.Alpha},
|
||||
LegacyResourceTrackerGC: {Default: true, PreRelease: featuregate.Alpha},
|
||||
EnableSuspendOnFailure: {Default: false, PreRelease: featuregate.Alpha},
|
||||
AuthenticateApplication: {Default: false, PreRelease: featuregate.Alpha},
|
||||
}
|
||||
|
||||
|
||||
@@ -28,18 +28,8 @@ import (
|
||||
wfContext "github.com/oam-dev/kubevela/pkg/workflow/context"
|
||||
)
|
||||
|
||||
const (
|
||||
// ReadyComponent is the key for depends on in workflow context
|
||||
ReadyComponent = "readyComponent__"
|
||||
)
|
||||
|
||||
// Input set data to parameter.
|
||||
func Input(ctx wfContext.Context, paramValue *value.Value, step v1beta1.WorkflowStep) error {
|
||||
for _, depend := range step.DependsOn {
|
||||
if _, err := ctx.GetVar(ReadyComponent, depend); err != nil {
|
||||
return errors.WithMessagef(err, "the depends on component [%s] is not ready", depend)
|
||||
}
|
||||
}
|
||||
for _, input := range step.Inputs {
|
||||
inputValue, err := ctx.GetVar(strings.Split(input.From, ".")...)
|
||||
if err != nil {
|
||||
@@ -66,13 +56,6 @@ func Output(ctx wfContext.Context, taskValue *value.Value, step v1beta1.Workflow
|
||||
if err := json.Unmarshal(js, &o); err != nil {
|
||||
return err
|
||||
}
|
||||
ready, err := value.NewValue(`true`, nil, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.SetVar(ready, ReadyComponent, o.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, output := range step.Outputs {
|
||||
|
||||
@@ -43,10 +43,6 @@ func TestInput(t *testing.T) {
|
||||
r.NoError(err)
|
||||
err = wfCtx.SetVar(score, "foo")
|
||||
r.NoError(err)
|
||||
ready, err := value.NewValue(`true`, nil, "")
|
||||
r.NoError(err)
|
||||
err = wfCtx.SetVar(ready, ReadyComponent, "mystep")
|
||||
r.NoError(err)
|
||||
err = Input(wfCtx, paramValue, v1beta1.WorkflowStep{
|
||||
DependsOn: []string{"mystep"},
|
||||
Inputs: common.StepInputs{{
|
||||
@@ -85,12 +81,6 @@ output: score: 99
|
||||
s, err := result.String()
|
||||
r.NoError(err)
|
||||
r.Equal(s, `99
|
||||
`)
|
||||
ready, err := wfCtx.GetVar(ReadyComponent, "mystep")
|
||||
r.NoError(err)
|
||||
s, err = ready.String()
|
||||
r.NoError(err)
|
||||
r.Equal(s, `true
|
||||
`)
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"github.com/pkg/errors"
|
||||
"k8s.io/apiserver/pkg/util/feature"
|
||||
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
|
||||
@@ -32,6 +33,7 @@ import (
|
||||
"github.com/oam-dev/kubevela/pkg/cue/model/value"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/packages"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/process"
|
||||
"github.com/oam-dev/kubevela/pkg/features"
|
||||
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
|
||||
wfContext "github.com/oam-dev/kubevela/pkg/workflow/context"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/hooks"
|
||||
@@ -47,6 +49,8 @@ var (
|
||||
const (
|
||||
// StatusReasonWait is the reason of the workflow progress condition which is Wait.
|
||||
StatusReasonWait = "Wait"
|
||||
// StatusReasonSkip is the reason of the workflow progress condition which is Skip.
|
||||
StatusReasonSkip = "Skip"
|
||||
// StatusReasonRendering is the reason of the workflow progress condition which is Rendering.
|
||||
StatusReasonRendering = "Rendering"
|
||||
// StatusReasonExecute is the reason of the workflow progress condition which is Execute.
|
||||
@@ -59,6 +63,8 @@ const (
|
||||
StatusReasonParameter = "ProcessParameter"
|
||||
// StatusReasonOutput is the reason of the workflow progress condition which is Output.
|
||||
StatusReasonOutput = "Output"
|
||||
// StatusReasonFailedAfterRetries is the reason of the workflow progress condition which is FailedAfterRetries.
|
||||
StatusReasonFailedAfterRetries = "FailedAfterRetries"
|
||||
)
|
||||
|
||||
// LoadTaskTemplate gets the workflowStep definition from cluster and resolve it.
|
||||
@@ -85,7 +91,8 @@ func (t *TaskLoader) GetTaskGenerator(ctx context.Context, name string) (wfTypes
|
||||
type taskRunner struct {
|
||||
name string
|
||||
run func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error)
|
||||
checkPending func(ctx wfContext.Context) bool
|
||||
checkPending func(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool
|
||||
skip func(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool)
|
||||
}
|
||||
|
||||
// Name return step name.
|
||||
@@ -99,10 +106,15 @@ func (tr *taskRunner) Run(ctx wfContext.Context, options *wfTypes.TaskRunOptions
|
||||
}
|
||||
|
||||
// Pending check task should be executed or not.
|
||||
func (tr *taskRunner) Pending(ctx wfContext.Context) bool {
|
||||
return tr.checkPending(ctx)
|
||||
func (tr *taskRunner) Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
|
||||
return tr.checkPending(ctx, stepStatus)
|
||||
}
|
||||
|
||||
func (tr *taskRunner) Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
|
||||
return tr.skip(dependsOnPhase, stepStatus)
|
||||
}
|
||||
|
||||
// nolint:gocyclo
|
||||
func (t *TaskLoader) makeTaskGenerator(templ string) (wfTypes.TaskGenerator, error) {
|
||||
return func(wfStep v1beta1.WorkflowStep, genOpt *wfTypes.GeneratorOptions) (wfTypes.TaskRunner, error) {
|
||||
|
||||
@@ -141,18 +153,21 @@ func (t *TaskLoader) makeTaskGenerator(templ string) (wfTypes.TaskGenerator, err
|
||||
|
||||
tRunner := new(taskRunner)
|
||||
tRunner.name = wfStep.Name
|
||||
tRunner.checkPending = func(ctx wfContext.Context) bool {
|
||||
for _, depend := range wfStep.DependsOn {
|
||||
if _, err := ctx.GetVar(hooks.ReadyComponent, depend); err != nil {
|
||||
return true
|
||||
}
|
||||
tRunner.checkPending = func(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
|
||||
return CheckPending(ctx, wfStep, stepStatus)
|
||||
}
|
||||
tRunner.skip = func(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
|
||||
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
|
||||
return exec.status(), false
|
||||
}
|
||||
for _, input := range wfStep.Inputs {
|
||||
if _, err := ctx.GetVar(strings.Split(input.From, ".")...); err != nil {
|
||||
return true
|
||||
}
|
||||
skip := SkipTaskRunner(&SkipOptions{
|
||||
If: wfStep.If,
|
||||
DependsOnPhase: dependsOnPhase,
|
||||
})
|
||||
if skip {
|
||||
exec.Skip("")
|
||||
}
|
||||
return false
|
||||
return exec.status(), skip
|
||||
}
|
||||
tRunner.run = func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error) {
|
||||
if options.GetTracer == nil {
|
||||
@@ -261,6 +276,7 @@ type executor struct {
|
||||
terminated bool
|
||||
failedAfterRetries bool
|
||||
wait bool
|
||||
skip bool
|
||||
|
||||
tracer monitorContext.Context
|
||||
}
|
||||
@@ -289,6 +305,13 @@ func (exec *executor) Wait(message string) {
|
||||
exec.wfStatus.Message = message
|
||||
}
|
||||
|
||||
func (exec *executor) Skip(message string) {
|
||||
exec.skip = true
|
||||
exec.wfStatus.Phase = common.WorkflowStepPhaseSkipped
|
||||
exec.wfStatus.Reason = StatusReasonSkip
|
||||
exec.wfStatus.Message = message
|
||||
}
|
||||
|
||||
func (exec *executor) err(ctx wfContext.Context, err error, reason string) {
|
||||
exec.wait = true
|
||||
exec.wfStatus.Phase = common.WorkflowStepPhaseFailed
|
||||
@@ -302,6 +325,7 @@ func (exec *executor) checkErrorTimes(ctx wfContext.Context) {
|
||||
if times >= MaxWorkflowStepErrorRetryTimes {
|
||||
exec.wait = false
|
||||
exec.failedAfterRetries = true
|
||||
exec.wfStatus.Reason = StatusReasonFailedAfterRetries
|
||||
}
|
||||
}
|
||||
|
||||
@@ -441,3 +465,58 @@ func NewTaskLoader(lt LoadTaskTemplate, pkgDiscover *packages.PackageDiscover, h
|
||||
logLevel: logLevel,
|
||||
}
|
||||
}
|
||||
|
||||
// SkipOptions is the options of skip task runner
|
||||
type SkipOptions struct {
|
||||
If string
|
||||
DependsOnPhase common.WorkflowStepPhase
|
||||
}
|
||||
|
||||
// SkipTaskRunner will decide whether to skip task runner.
|
||||
func SkipTaskRunner(options *SkipOptions) bool {
|
||||
switch options.If {
|
||||
case "always":
|
||||
return false
|
||||
case "":
|
||||
return options.DependsOnPhase != common.WorkflowStepPhaseSucceeded
|
||||
default:
|
||||
// TODO:(fog) support more if cases
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// CheckPending checks whether to pending task run
|
||||
func CheckPending(ctx wfContext.Context, step v1beta1.WorkflowStep, stepStatus map[string]common.StepStatus) bool {
|
||||
for _, depend := range step.DependsOn {
|
||||
if status, ok := stepStatus[depend]; ok {
|
||||
if !IsStepFinish(status.Phase, status.Reason) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, input := range step.Inputs {
|
||||
if _, err := ctx.GetVar(strings.Split(input.From, ".")...); err != nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsStepFinish will decide whether step is finish.
|
||||
func IsStepFinish(phase common.WorkflowStepPhase, reason string) bool {
|
||||
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
|
||||
return phase == common.WorkflowStepPhaseSucceeded
|
||||
}
|
||||
switch phase {
|
||||
case common.WorkflowStepPhaseFailed:
|
||||
return reason == StatusReasonTerminate || reason == StatusReasonFailedAfterRetries
|
||||
case common.WorkflowStepPhaseSkipped:
|
||||
return true
|
||||
case common.WorkflowStepPhaseSucceeded:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/oam-dev/kubevela/pkg/cue/model/value"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/process"
|
||||
wfContext "github.com/oam-dev/kubevela/pkg/workflow/context"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/hooks"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/providers"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/types"
|
||||
)
|
||||
@@ -270,6 +269,7 @@ close({
|
||||
r.Equal(operation.Waiting, false)
|
||||
r.Equal(operation.FailedAfterRetries, true)
|
||||
r.Equal(status.Phase, common.WorkflowStepPhaseFailed)
|
||||
r.Equal(status.Reason, StatusReasonFailedAfterRetries)
|
||||
default:
|
||||
r.Equal(operation.Waiting, true)
|
||||
r.Equal(status.Phase, common.WorkflowStepPhaseFailed)
|
||||
@@ -438,14 +438,14 @@ func TestPendingInputCheck(t *testing.T) {
|
||||
r.NoError(err)
|
||||
run, err := gen(step, &types.GeneratorOptions{})
|
||||
r.NoError(err)
|
||||
r.Equal(run.Pending(wfCtx), true)
|
||||
r.Equal(run.Pending(wfCtx, nil), true)
|
||||
score, err := value.NewValue(`
|
||||
100
|
||||
`, nil, "")
|
||||
r.NoError(err)
|
||||
err = wfCtx.SetVar(score, "score")
|
||||
r.NoError(err)
|
||||
r.Equal(run.Pending(wfCtx), false)
|
||||
r.Equal(run.Pending(wfCtx, nil), false)
|
||||
}
|
||||
|
||||
func TestPendingDependsOnCheck(t *testing.T) {
|
||||
@@ -473,12 +473,49 @@ func TestPendingDependsOnCheck(t *testing.T) {
|
||||
r.NoError(err)
|
||||
run, err := gen(step, &types.GeneratorOptions{})
|
||||
r.NoError(err)
|
||||
r.Equal(run.Pending(wfCtx), true)
|
||||
ready, err := value.NewValue("true", nil, "")
|
||||
r.Equal(run.Pending(wfCtx, nil), true)
|
||||
ss := map[string]common.StepStatus{
|
||||
"depend": {
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}
|
||||
r.Equal(run.Pending(wfCtx, ss), false)
|
||||
}
|
||||
|
||||
func TestSkip(t *testing.T) {
|
||||
r := require.New(t)
|
||||
discover := providers.NewProviders()
|
||||
discover.Register("test", map[string]providers.Handler{
|
||||
"ok": func(ctx wfContext.Context, v *value.Value, act types.Action) error {
|
||||
return nil
|
||||
},
|
||||
})
|
||||
step := v1beta1.WorkflowStep{
|
||||
Name: "skip",
|
||||
Type: "ok",
|
||||
}
|
||||
pCtx := process.NewContext(process.ContextData{
|
||||
AppName: "myapp",
|
||||
CompName: "mycomp",
|
||||
Namespace: "default",
|
||||
AppRevisionName: "myapp-v1",
|
||||
})
|
||||
tasksLoader := NewTaskLoader(mockLoadTemplate, nil, discover, 0, pCtx)
|
||||
gen, err := tasksLoader.GetTaskGenerator(context.Background(), step.Type)
|
||||
r.NoError(err)
|
||||
err = wfCtx.SetVar(ready, hooks.ReadyComponent, "depend")
|
||||
runner, err := gen(step, &types.GeneratorOptions{})
|
||||
r.NoError(err)
|
||||
r.Equal(run.Pending(wfCtx), false)
|
||||
status, skip := runner.Skip(common.WorkflowStepPhaseFailed, nil)
|
||||
r.Equal(skip, true)
|
||||
r.Equal(status.Phase, common.WorkflowStepPhaseSkipped)
|
||||
r.Equal(status.Reason, StatusReasonSkip)
|
||||
runner2, err := gen(v1beta1.WorkflowStep{
|
||||
If: "always",
|
||||
Name: "test",
|
||||
}, &types.GeneratorOptions{ID: "124"})
|
||||
r.NoError(err)
|
||||
_, skip = runner2.Skip(common.WorkflowStepPhaseFailed, nil)
|
||||
r.Equal(skip, false)
|
||||
}
|
||||
|
||||
func newWorkflowContextForTest(t *testing.T) wfContext.Context {
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
builtintime "time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"k8s.io/apiserver/pkg/util/feature"
|
||||
"k8s.io/client-go/rest"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
@@ -29,6 +30,7 @@ import (
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/packages"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/process"
|
||||
"github.com/oam-dev/kubevela/pkg/features"
|
||||
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
|
||||
"github.com/oam-dev/kubevela/pkg/oam/discoverymapper"
|
||||
"github.com/oam-dev/kubevela/pkg/velaql/providers/query"
|
||||
@@ -73,7 +75,7 @@ func (td *taskDiscover) GetTaskGenerator(ctx context.Context, name string) (type
|
||||
func suspend(step v1beta1.WorkflowStep, opt *types.GeneratorOptions) (types.TaskRunner, error) {
|
||||
tr := &suspendTaskRunner{
|
||||
id: opt.ID,
|
||||
name: step.Name,
|
||||
step: step,
|
||||
wait: false,
|
||||
}
|
||||
|
||||
@@ -92,6 +94,7 @@ func StepGroup(step v1beta1.WorkflowStep, opt *types.GeneratorOptions) (types.Ta
|
||||
return &stepGroupTaskRunner{
|
||||
id: opt.ID,
|
||||
name: step.Name,
|
||||
step: step,
|
||||
subTaskRunners: opt.SubTaskRunners,
|
||||
}, nil
|
||||
}
|
||||
@@ -119,40 +122,64 @@ func NewTaskDiscoverFromRevision(ctx monitorContext.Context, providerHandlers pr
|
||||
}
|
||||
|
||||
type suspendTaskRunner struct {
|
||||
id string
|
||||
name string
|
||||
wait bool
|
||||
id string
|
||||
step v1beta1.WorkflowStep
|
||||
wait bool
|
||||
phase common.WorkflowStepPhase
|
||||
}
|
||||
|
||||
// Name return suspend step name.
|
||||
func (tr *suspendTaskRunner) Name() string {
|
||||
return tr.name
|
||||
return tr.step.Name
|
||||
}
|
||||
|
||||
// Run make workflow suspend.
|
||||
func (tr *suspendTaskRunner) Run(ctx wfContext.Context, options *types.TaskRunOptions) (common.StepStatus, *types.Operation, error) {
|
||||
if tr.wait {
|
||||
tr.phase = common.WorkflowStepPhaseRunning
|
||||
} else {
|
||||
tr.phase = common.WorkflowStepPhaseSucceeded
|
||||
}
|
||||
stepStatus := common.StepStatus{
|
||||
ID: tr.id,
|
||||
Name: tr.name,
|
||||
Name: tr.step.Name,
|
||||
Type: types.WorkflowStepTypeSuspend,
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
}
|
||||
|
||||
if tr.wait {
|
||||
stepStatus.Phase = common.WorkflowStepPhaseRunning
|
||||
Phase: tr.phase,
|
||||
}
|
||||
|
||||
return stepStatus, &types.Operation{Suspend: true}, nil
|
||||
}
|
||||
|
||||
// Pending check task should be executed or not.
|
||||
func (tr *suspendTaskRunner) Pending(ctx wfContext.Context) bool {
|
||||
return false
|
||||
func (tr *suspendTaskRunner) Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
|
||||
return custom.CheckPending(ctx, tr.step, stepStatus)
|
||||
}
|
||||
|
||||
func (tr *suspendTaskRunner) Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
|
||||
status := common.StepStatus{
|
||||
ID: tr.id,
|
||||
Name: tr.step.Name,
|
||||
Type: types.WorkflowStepTypeSuspend,
|
||||
Phase: tr.phase,
|
||||
}
|
||||
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
|
||||
return status, false
|
||||
}
|
||||
skip := custom.SkipTaskRunner(&custom.SkipOptions{
|
||||
If: tr.step.If,
|
||||
DependsOnPhase: dependsOnPhase,
|
||||
})
|
||||
if skip {
|
||||
status.Phase = common.WorkflowStepPhaseSkipped
|
||||
status.Reason = custom.StatusReasonSkip
|
||||
}
|
||||
return status, skip
|
||||
}
|
||||
|
||||
type stepGroupTaskRunner struct {
|
||||
id string
|
||||
name string
|
||||
step v1beta1.WorkflowStep
|
||||
subTaskRunners []types.TaskRunner
|
||||
}
|
||||
|
||||
@@ -161,12 +188,43 @@ func (tr *stepGroupTaskRunner) Name() string {
|
||||
return tr.name
|
||||
}
|
||||
|
||||
// Pending check task should be executed or not.
|
||||
func (tr *stepGroupTaskRunner) Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
|
||||
return custom.CheckPending(ctx, tr.step, stepStatus)
|
||||
}
|
||||
|
||||
func (tr *stepGroupTaskRunner) Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
|
||||
status := common.StepStatus{
|
||||
ID: tr.id,
|
||||
Name: tr.step.Name,
|
||||
Type: types.WorkflowStepTypeStepGroup,
|
||||
}
|
||||
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
|
||||
return status, false
|
||||
}
|
||||
skip := custom.SkipTaskRunner(&custom.SkipOptions{
|
||||
If: tr.step.If,
|
||||
DependsOnPhase: dependsOnPhase,
|
||||
})
|
||||
if skip {
|
||||
status.Phase = common.WorkflowStepPhaseSkipped
|
||||
status.Reason = custom.StatusReasonSkip
|
||||
stepStatus[tr.step.Name] = common.StepStatus{
|
||||
ID: tr.id,
|
||||
Phase: status.Phase,
|
||||
}
|
||||
// return false here to set all the sub steps to skipped
|
||||
return status, false
|
||||
}
|
||||
return status, skip
|
||||
}
|
||||
|
||||
// Run make workflow step group.
|
||||
func (tr *stepGroupTaskRunner) Run(ctx wfContext.Context, options *types.TaskRunOptions) (common.StepStatus, *types.Operation, error) {
|
||||
e := options.Engine
|
||||
if len(tr.subTaskRunners) > 0 {
|
||||
// set sub steps to dag mode for now
|
||||
e.SetParentRunner(tr.name)
|
||||
// set sub steps to dag mode for now
|
||||
if err := e.Run(tr.subTaskRunners, true); err != nil {
|
||||
return common.StepStatus{
|
||||
ID: tr.id,
|
||||
@@ -178,34 +236,39 @@ func (tr *stepGroupTaskRunner) Run(ctx wfContext.Context, options *types.TaskRun
|
||||
e.SetParentRunner("")
|
||||
}
|
||||
stepStatus := e.GetStepStatus(tr.name)
|
||||
var phase common.WorkflowStepPhase
|
||||
subStepPhases := make(map[common.WorkflowStepPhase]int)
|
||||
status := common.StepStatus{
|
||||
ID: tr.id,
|
||||
Name: tr.name,
|
||||
Type: types.WorkflowStepTypeStepGroup,
|
||||
}
|
||||
|
||||
subStepCounts := make(map[string]int)
|
||||
for _, subStepsStatus := range stepStatus.SubStepsStatus {
|
||||
subStepPhases[subStepsStatus.Phase]++
|
||||
subStepCounts[string(subStepsStatus.Phase)]++
|
||||
subStepCounts[subStepsStatus.Reason]++
|
||||
}
|
||||
switch {
|
||||
case len(stepStatus.SubStepsStatus) < len(tr.subTaskRunners):
|
||||
phase = common.WorkflowStepPhaseRunning
|
||||
case subStepPhases[common.WorkflowStepPhaseRunning] > 0:
|
||||
phase = common.WorkflowStepPhaseRunning
|
||||
case subStepPhases[common.WorkflowStepPhaseStopped] > 0:
|
||||
phase = common.WorkflowStepPhaseStopped
|
||||
case subStepPhases[common.WorkflowStepPhaseFailed] > 0:
|
||||
phase = common.WorkflowStepPhaseFailed
|
||||
status.Phase = common.WorkflowStepPhaseRunning
|
||||
case subStepCounts[string(common.WorkflowStepPhaseRunning)] > 0:
|
||||
status.Phase = common.WorkflowStepPhaseRunning
|
||||
case subStepCounts[string(common.WorkflowStepPhaseStopped)] > 0:
|
||||
status.Phase = common.WorkflowStepPhaseStopped
|
||||
case subStepCounts[string(common.WorkflowStepPhaseFailed)] > 0:
|
||||
status.Phase = common.WorkflowStepPhaseFailed
|
||||
switch {
|
||||
case subStepCounts[custom.StatusReasonFailedAfterRetries] > 0:
|
||||
status.Reason = custom.StatusReasonFailedAfterRetries
|
||||
case subStepCounts[custom.StatusReasonTerminate] > 0:
|
||||
status.Reason = custom.StatusReasonTerminate
|
||||
}
|
||||
case subStepCounts[string(common.WorkflowStepPhaseSkipped)] > 0:
|
||||
status.Phase = common.WorkflowStepPhaseSkipped
|
||||
status.Reason = custom.StatusReasonSkip
|
||||
default:
|
||||
phase = common.WorkflowStepPhaseSucceeded
|
||||
status.Phase = common.WorkflowStepPhaseSucceeded
|
||||
}
|
||||
return common.StepStatus{
|
||||
ID: tr.id,
|
||||
Name: tr.name,
|
||||
Type: types.WorkflowStepTypeStepGroup,
|
||||
Phase: phase,
|
||||
}, e.GetOperation(), nil
|
||||
}
|
||||
|
||||
// Pending check task should be executed or not.
|
||||
func (tr *stepGroupTaskRunner) Pending(ctx wfContext.Context) bool {
|
||||
return false
|
||||
return status, e.GetOperation(), nil
|
||||
}
|
||||
|
||||
// NewViewTaskDiscover will create a client for load task generator.
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
|
||||
|
||||
"github.com/oam-dev/kubevela/pkg/cue/process"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/tasks/custom"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/types"
|
||||
@@ -82,10 +83,36 @@ func TestSuspendStep(t *testing.T) {
|
||||
}
|
||||
gen, err := discover.GetTaskGenerator(context.Background(), "suspend")
|
||||
r.NoError(err)
|
||||
runner, err := gen(v1beta1.WorkflowStep{Name: "test"}, &types.GeneratorOptions{ID: "124"})
|
||||
runner, err := gen(v1beta1.WorkflowStep{
|
||||
Name: "test",
|
||||
DependsOn: []string{"depend"},
|
||||
}, &types.GeneratorOptions{ID: "124"})
|
||||
r.NoError(err)
|
||||
r.Equal(runner.Name(), "test")
|
||||
r.Equal(runner.Pending(nil), false)
|
||||
|
||||
// test pending
|
||||
r.Equal(runner.Pending(nil, nil), true)
|
||||
ss := map[string]common.StepStatus{
|
||||
"depend": {
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}
|
||||
r.Equal(runner.Pending(nil, ss), false)
|
||||
|
||||
// test skip
|
||||
status, skip := runner.Skip(common.WorkflowStepPhaseFailed, nil)
|
||||
r.Equal(skip, true)
|
||||
r.Equal(status.Phase, common.WorkflowStepPhaseSkipped)
|
||||
r.Equal(status.Reason, custom.StatusReasonSkip)
|
||||
runner2, err := gen(v1beta1.WorkflowStep{
|
||||
If: "always",
|
||||
Name: "test",
|
||||
}, &types.GeneratorOptions{ID: "124"})
|
||||
r.NoError(err)
|
||||
_, skip = runner2.Skip(common.WorkflowStepPhaseFailed, nil)
|
||||
r.Equal(skip, false)
|
||||
|
||||
// test run
|
||||
status, act, err := runner.Run(nil, nil)
|
||||
r.NoError(err)
|
||||
r.Equal(act.Suspend, true)
|
||||
@@ -127,11 +154,38 @@ func TestStepGroupStep(t *testing.T) {
|
||||
r.NoError(err)
|
||||
gen, err := discover.GetTaskGenerator(context.Background(), "stepGroup")
|
||||
r.NoError(err)
|
||||
runner, err := gen(v1beta1.WorkflowStep{Name: "test"}, &types.GeneratorOptions{ID: "124", SubTaskRunners: []types.TaskRunner{subRunner}})
|
||||
runner, err := gen(v1beta1.WorkflowStep{
|
||||
Name: "test",
|
||||
DependsOn: []string{"depend"},
|
||||
}, &types.GeneratorOptions{ID: "124", SubTaskRunners: []types.TaskRunner{subRunner}})
|
||||
r.NoError(err)
|
||||
r.Equal(runner.Name(), "test")
|
||||
r.Equal(runner.Pending(nil), false)
|
||||
|
||||
// test pending
|
||||
r.Equal(runner.Pending(nil, nil), true)
|
||||
ss := map[string]common.StepStatus{
|
||||
"depend": {
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}
|
||||
r.Equal(runner.Pending(nil, ss), false)
|
||||
|
||||
// test skip
|
||||
stepStatus := make(map[string]common.StepStatus)
|
||||
status, skip := runner.Skip(common.WorkflowStepPhaseFailed, stepStatus)
|
||||
r.Equal(skip, false)
|
||||
r.Equal(stepStatus["test"].Phase, common.WorkflowStepPhaseSkipped)
|
||||
r.Equal(status.Phase, common.WorkflowStepPhaseSkipped)
|
||||
r.Equal(status.Reason, custom.StatusReasonSkip)
|
||||
runner2, err := gen(v1beta1.WorkflowStep{
|
||||
If: "always",
|
||||
Name: "test",
|
||||
}, &types.GeneratorOptions{ID: "124"})
|
||||
r.NoError(err)
|
||||
_, skip = runner2.Skip(common.WorkflowStepPhaseFailed, stepStatus)
|
||||
r.Equal(skip, false)
|
||||
|
||||
// test run
|
||||
testCases := []struct {
|
||||
name string
|
||||
engine *testEngine
|
||||
|
||||
@@ -30,8 +30,9 @@ import (
|
||||
// TaskRunner is a task runner.
|
||||
type TaskRunner interface {
|
||||
Name() string
|
||||
Pending(ctx wfContext.Context) bool
|
||||
Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool
|
||||
Run(ctx wfContext.Context, options *TaskRunOptions) (common.StepStatus, *Operation, error)
|
||||
Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool)
|
||||
}
|
||||
|
||||
// TaskDiscover is the interface to obtain the TaskGenerator。
|
||||
@@ -71,6 +72,7 @@ type Operation struct {
|
||||
Suspend bool
|
||||
Terminated bool
|
||||
Waiting bool
|
||||
Skip bool
|
||||
FailedAfterRetries bool
|
||||
}
|
||||
|
||||
|
||||
+236
-118
@@ -25,6 +25,7 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apiserver/pkg/util/feature"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
|
||||
@@ -32,6 +33,7 @@ import (
|
||||
oamcore "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
|
||||
"github.com/oam-dev/kubevela/pkg/controller/utils"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/model/value"
|
||||
"github.com/oam-dev/kubevela/pkg/features"
|
||||
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
|
||||
"github.com/oam-dev/kubevela/pkg/monitor/metrics"
|
||||
"github.com/oam-dev/kubevela/pkg/oam"
|
||||
@@ -41,6 +43,7 @@ import (
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/debug"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/recorder"
|
||||
wfTasks "github.com/oam-dev/kubevela/pkg/workflow/tasks"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/tasks/custom"
|
||||
wfTypes "github.com/oam-dev/kubevela/pkg/workflow/types"
|
||||
)
|
||||
|
||||
@@ -61,8 +64,10 @@ const (
|
||||
// backoffTimeCoefficient is the coefficient of time to wait before reconcile workflow again
|
||||
backoffTimeCoefficient = 0.05
|
||||
|
||||
// MessageFailedAfterRetries is the message of failed after retries
|
||||
MessageFailedAfterRetries = "The workflow suspends automatically because the failed times of steps have reached the limit"
|
||||
// MessageTerminatedFailedAfterRetries is the message of failed after retries
|
||||
MessageTerminatedFailedAfterRetries = "The workflow terminates automatically because the failed times of steps have reached the limit"
|
||||
// MessageSuspendFailedAfterRetries is the message of failed after retries
|
||||
MessageSuspendFailedAfterRetries = "The workflow suspends automatically because the failed times of steps have reached the limit"
|
||||
// MessageInitializingWorkflow is the message of initializing workflow
|
||||
MessageInitializingWorkflow = "Initializing workflow"
|
||||
)
|
||||
@@ -103,56 +108,24 @@ func (w *workflow) ExecuteSteps(ctx monitorContext.Context, appRev *oamcore.Appl
|
||||
}
|
||||
|
||||
if w.app.Status.Workflow == nil || w.app.Status.Workflow.AppRevision != revAndSpecHash {
|
||||
ctx.Info("Restart Workflow")
|
||||
status := w.app.Status.Workflow
|
||||
if status != nil && !status.Finished {
|
||||
status.Terminated = true
|
||||
return common.WorkflowStateTerminated, nil
|
||||
}
|
||||
w.app.Status.Workflow = &common.WorkflowStatus{
|
||||
AppRevision: revAndSpecHash,
|
||||
Mode: common.WorkflowModeStep,
|
||||
StartTime: metav1.Now(),
|
||||
}
|
||||
w.app.Status.Workflow.Message = MessageInitializingWorkflow
|
||||
if w.dagMode {
|
||||
w.app.Status.Workflow.Mode = common.WorkflowModeDAG
|
||||
}
|
||||
// clean recorded resources info.
|
||||
w.app.Status.Services = nil
|
||||
w.app.Status.AppliedResources = nil
|
||||
|
||||
// clean conditions after render
|
||||
var reservedConditions []condition.Condition
|
||||
for i, cond := range w.app.Status.Conditions {
|
||||
condTpy, err := common.ParseApplicationConditionType(string(cond.Type))
|
||||
if err == nil {
|
||||
if condTpy <= common.RenderCondition {
|
||||
reservedConditions = append(reservedConditions, w.app.Status.Conditions[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
w.app.Status.Conditions = reservedConditions
|
||||
StepStatusCache.Delete(fmt.Sprintf("%s-%s", w.app.Name, w.app.Namespace))
|
||||
wfContext.CleanupMemoryStore(w.app.Name, w.app.Namespace)
|
||||
return common.WorkflowStateInitializing, nil
|
||||
return w.restartWorkflow(ctx, revAndSpecHash)
|
||||
}
|
||||
|
||||
wfStatus := w.app.Status.Workflow
|
||||
cacheKey := fmt.Sprintf("%s-%s", w.app.Name, w.app.Namespace)
|
||||
|
||||
allTasksDone, allTasksSucceeded := w.allDone(taskRunners)
|
||||
if wfStatus.Finished {
|
||||
StepStatusCache.Delete(cacheKey)
|
||||
return common.WorkflowStateFinished, nil
|
||||
}
|
||||
if wfStatus.Terminated {
|
||||
if checkWorkflowTerminated(wfStatus, allTasksDone) {
|
||||
return common.WorkflowStateTerminated, nil
|
||||
}
|
||||
if wfStatus.Suspend {
|
||||
return common.WorkflowStateSuspended, nil
|
||||
}
|
||||
allTasksDone := w.allDone(taskRunners)
|
||||
if allTasksDone {
|
||||
if allTasksSucceeded {
|
||||
return common.WorkflowStateSucceeded, nil
|
||||
}
|
||||
|
||||
@@ -171,15 +144,7 @@ func (w *workflow) ExecuteSteps(ctx monitorContext.Context, appRev *oamcore.Appl
|
||||
}
|
||||
}
|
||||
|
||||
e := &engine{
|
||||
status: wfStatus,
|
||||
monitorCtx: ctx,
|
||||
app: w.app,
|
||||
wfCtx: wfCtx,
|
||||
cli: w.cli,
|
||||
debug: w.debug,
|
||||
rk: w.rk,
|
||||
}
|
||||
e := newEngine(ctx, wfCtx, w, wfStatus)
|
||||
|
||||
err = e.Run(taskRunners, w.dagMode)
|
||||
if err != nil {
|
||||
@@ -191,15 +156,19 @@ func (w *workflow) ExecuteSteps(ctx monitorContext.Context, appRev *oamcore.Appl
|
||||
|
||||
e.checkWorkflowStatusMessage(wfStatus)
|
||||
StepStatusCache.Store(cacheKey, len(wfStatus.Steps))
|
||||
allTasksDone, allTasksSucceeded = w.allDone(taskRunners)
|
||||
if wfStatus.Terminated {
|
||||
wfContext.CleanupMemoryStore(e.app.Name, e.app.Namespace)
|
||||
return common.WorkflowStateTerminated, nil
|
||||
e.cleanBackoffTimesForTerminated()
|
||||
if checkWorkflowTerminated(wfStatus, allTasksDone) {
|
||||
wfContext.CleanupMemoryStore(e.app.Name, e.app.Namespace)
|
||||
return common.WorkflowStateTerminated, nil
|
||||
}
|
||||
}
|
||||
if wfStatus.Suspend {
|
||||
wfContext.CleanupMemoryStore(e.app.Name, e.app.Namespace)
|
||||
return common.WorkflowStateSuspended, nil
|
||||
}
|
||||
if w.allDone(taskRunners) {
|
||||
if allTasksSucceeded {
|
||||
wfStatus.Message = string(common.WorkflowStateSucceeded)
|
||||
return common.WorkflowStateSucceeded, nil
|
||||
}
|
||||
@@ -207,6 +176,88 @@ func (w *workflow) ExecuteSteps(ctx monitorContext.Context, appRev *oamcore.Appl
|
||||
return common.WorkflowStateExecuting, nil
|
||||
}
|
||||
|
||||
func checkWorkflowTerminated(wfStatus *common.WorkflowStatus, allTasksDone bool) bool {
|
||||
return (wfStatus.Terminated && allTasksDone) || (wfStatus.Terminated && wfStatus.Suspend)
|
||||
}
|
||||
|
||||
func (w *workflow) restartWorkflow(ctx monitorContext.Context, revAndSpecHash string) (common.WorkflowState, error) {
|
||||
ctx.Info("Restart Workflow")
|
||||
status := w.app.Status.Workflow
|
||||
if status != nil && !status.Finished {
|
||||
status.Terminated = true
|
||||
return common.WorkflowStateTerminated, nil
|
||||
}
|
||||
w.app.Status.Workflow = &common.WorkflowStatus{
|
||||
AppRevision: revAndSpecHash,
|
||||
Mode: common.WorkflowModeStep,
|
||||
StartTime: metav1.Now(),
|
||||
}
|
||||
w.app.Status.Workflow.Message = MessageInitializingWorkflow
|
||||
if w.dagMode {
|
||||
w.app.Status.Workflow.Mode = common.WorkflowModeDAG
|
||||
}
|
||||
// clean recorded resources info.
|
||||
w.app.Status.Services = nil
|
||||
w.app.Status.AppliedResources = nil
|
||||
|
||||
// clean conditions after render
|
||||
var reservedConditions []condition.Condition
|
||||
for i, cond := range w.app.Status.Conditions {
|
||||
condTpy, err := common.ParseApplicationConditionType(string(cond.Type))
|
||||
if err == nil {
|
||||
if condTpy <= common.RenderCondition {
|
||||
reservedConditions = append(reservedConditions, w.app.Status.Conditions[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
w.app.Status.Conditions = reservedConditions
|
||||
StepStatusCache.Delete(fmt.Sprintf("%s-%s", w.app.Name, w.app.Namespace))
|
||||
wfContext.CleanupMemoryStore(w.app.Name, w.app.Namespace)
|
||||
return common.WorkflowStateInitializing, nil
|
||||
}
|
||||
|
||||
func newEngine(ctx monitorContext.Context, wfCtx wfContext.Context, w *workflow, wfStatus *common.WorkflowStatus) *engine {
|
||||
stepStatus := make(map[string]common.StepStatus)
|
||||
for _, ss := range wfStatus.Steps {
|
||||
stepStatus[ss.Name] = common.StepStatus{
|
||||
Phase: ss.Phase,
|
||||
ID: ss.ID,
|
||||
Reason: ss.Reason,
|
||||
}
|
||||
for _, sss := range ss.SubStepsStatus {
|
||||
stepStatus[sss.Name] = common.StepStatus{
|
||||
Phase: sss.Phase,
|
||||
ID: sss.ID,
|
||||
Reason: ss.Reason,
|
||||
}
|
||||
}
|
||||
}
|
||||
stepDependsOn := make(map[string][]string)
|
||||
if w.app.Spec.Workflow != nil {
|
||||
for _, step := range w.app.Spec.Workflow.Steps {
|
||||
stepDependsOn[step.Name] = append(stepDependsOn[step.Name], step.DependsOn...)
|
||||
for _, sub := range step.SubSteps {
|
||||
stepDependsOn[sub.Name] = append(stepDependsOn[sub.Name], sub.DependsOn...)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, comp := range w.app.Spec.Components {
|
||||
stepDependsOn[comp.Name] = append(stepDependsOn[comp.Name], comp.DependsOn...)
|
||||
}
|
||||
}
|
||||
return &engine{
|
||||
status: wfStatus,
|
||||
monitorCtx: ctx,
|
||||
app: w.app,
|
||||
wfCtx: wfCtx,
|
||||
cli: w.cli,
|
||||
debug: w.debug,
|
||||
rk: w.rk,
|
||||
stepStatus: stepStatus,
|
||||
stepDependsOn: stepDependsOn,
|
||||
}
|
||||
}
|
||||
|
||||
// Trace record the workflow execute history.
|
||||
func (w *workflow) Trace() error {
|
||||
if DisableRecorder {
|
||||
@@ -294,21 +345,23 @@ func (w *workflow) getWorkflowStepByName(name string) oamcore.WorkflowStep {
|
||||
return oamcore.WorkflowStep{}
|
||||
}
|
||||
|
||||
func (w *workflow) allDone(taskRunners []wfTypes.TaskRunner) bool {
|
||||
func (w *workflow) allDone(taskRunners []wfTypes.TaskRunner) (bool, bool) {
|
||||
success := true
|
||||
status := w.app.Status.Workflow
|
||||
for _, t := range taskRunners {
|
||||
done := false
|
||||
for _, ss := range status.Steps {
|
||||
if ss.Name == t.Name() {
|
||||
done = ss.Phase == common.WorkflowStepPhaseSucceeded
|
||||
done = custom.IsStepFinish(ss.Phase, ss.Reason)
|
||||
success = done && (ss.Phase == common.WorkflowStepPhaseSucceeded)
|
||||
break
|
||||
}
|
||||
}
|
||||
if !done {
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
}
|
||||
return true
|
||||
return true, success
|
||||
}
|
||||
|
||||
func (w *workflow) makeContext(appName string) (wfCtx wfContext.Context, err error) {
|
||||
@@ -432,18 +485,15 @@ func (e *engine) runAsDAG(taskRunners []wfTypes.TaskRunner) error {
|
||||
wfCtx := e.wfCtx
|
||||
done := true
|
||||
for _, tRunner := range taskRunners {
|
||||
ready := false
|
||||
finish := false
|
||||
var stepID string
|
||||
for _, ss := range e.status.Steps {
|
||||
if ss.Name == tRunner.Name() {
|
||||
stepID = ss.ID
|
||||
ready = ss.Phase == common.WorkflowStepPhaseSucceeded
|
||||
break
|
||||
}
|
||||
if status, ok := e.stepStatus[tRunner.Name()]; ok {
|
||||
stepID = status.ID
|
||||
finish = custom.IsStepFinish(status.Phase, status.Reason)
|
||||
}
|
||||
if !ready {
|
||||
if !finish {
|
||||
done = false
|
||||
if tRunner.Pending(wfCtx) {
|
||||
if tRunner.Pending(wfCtx, e.stepStatus) {
|
||||
pendingTasks = append(pendingTasks, tRunner)
|
||||
continue
|
||||
}
|
||||
@@ -479,45 +529,37 @@ func (e *engine) Run(taskRunners []wfTypes.TaskRunner, dag bool) error {
|
||||
if dag {
|
||||
err = e.runAsDAG(taskRunners)
|
||||
} else {
|
||||
err = e.steps(e.todoByIndex(taskRunners), dag)
|
||||
err = e.steps(taskRunners, dag)
|
||||
}
|
||||
|
||||
e.checkFailedAfterRetries()
|
||||
e.setNextExecuteTime()
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *engine) checkWorkflowStatusMessage(wfStatus *common.WorkflowStatus) {
|
||||
if !e.waiting && e.failedAfterRetries {
|
||||
e.status.Message = MessageFailedAfterRetries
|
||||
return
|
||||
}
|
||||
|
||||
if wfStatus.Terminated {
|
||||
switch {
|
||||
case !e.waiting && e.failedAfterRetries && feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure):
|
||||
e.status.Message = MessageSuspendFailedAfterRetries
|
||||
case e.failedAfterRetries && !feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure):
|
||||
e.status.Message = MessageTerminatedFailedAfterRetries
|
||||
case wfStatus.Terminated:
|
||||
e.status.Message = string(common.WorkflowStateTerminated)
|
||||
}
|
||||
if wfStatus.Suspend {
|
||||
case wfStatus.Suspend:
|
||||
e.status.Message = string(common.WorkflowStateSuspended)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (e *engine) todoByIndex(taskRunners []wfTypes.TaskRunner) []wfTypes.TaskRunner {
|
||||
index := 0
|
||||
for _, t := range taskRunners {
|
||||
for _, ss := range e.status.Steps {
|
||||
if ss.Name == t.Name() {
|
||||
if ss.Phase == common.WorkflowStepPhaseSucceeded {
|
||||
index++
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return taskRunners[index:]
|
||||
}
|
||||
|
||||
func (e *engine) steps(taskRunners []wfTypes.TaskRunner, dag bool) error {
|
||||
wfCtx := e.wfCtx
|
||||
for _, runner := range taskRunners {
|
||||
var err error
|
||||
for index, runner := range taskRunners {
|
||||
if status, ok := e.stepStatus[runner.Name()]; ok {
|
||||
if custom.IsStepFinish(status.Phase, status.Reason) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
options := &wfTypes.TaskRunOptions{
|
||||
GetTracer: func(id string, stepStatus oamcore.WorkflowStep) monitorContext.Context {
|
||||
return e.monitorCtx.Fork(id, monitorContext.DurationMetric(func(v float64) {
|
||||
@@ -535,42 +577,41 @@ func (e *engine) steps(taskRunners []wfTypes.TaskRunner, dag bool) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
status, operation, err := runner.Run(wfCtx, options)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
var status common.StepStatus
|
||||
operation := &wfTypes.Operation{}
|
||||
skip := false
|
||||
status, skip = runner.Skip(e.findDependPhase(taskRunners, index, dag), e.stepStatus)
|
||||
if !skip {
|
||||
status, operation, err = runner.Run(wfCtx, options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
e.updateStepStatus(status)
|
||||
|
||||
e.failedAfterRetries = e.failedAfterRetries || operation.FailedAfterRetries
|
||||
e.waiting = e.waiting || operation.Waiting
|
||||
if status.Phase == common.WorkflowStepPhaseSucceeded || (status.Phase == common.WorkflowStepPhaseRunning && status.Type == wfTypes.WorkflowStepTypeSuspend) {
|
||||
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
|
||||
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffReason, status.ID)
|
||||
if err := wfCtx.Commit(); err != nil {
|
||||
return errors.WithMessage(err, "commit workflow context")
|
||||
// for the suspend step with duration, there's no need to increase the backoff time in reconcile when it's still running
|
||||
if !custom.IsStepFinish(status.Phase, status.Reason) && !isWaitSuspendStep(status) {
|
||||
if err := handleBackoffTimes(wfCtx, status, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.finishStep(operation)
|
||||
if e.needStop() {
|
||||
return nil
|
||||
if dag {
|
||||
continue
|
||||
}
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
// clear the backoff time when the step is finished
|
||||
if err := handleBackoffTimes(wfCtx, status, true); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if val, exists := wfCtx.GetValueInMemory(wfTypes.ContextPrefixBackoffReason, status.ID); !exists || val != status.Message {
|
||||
wfCtx.SetValueInMemory(status.Message, wfTypes.ContextPrefixBackoffReason, status.ID)
|
||||
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
|
||||
e.finishStep(operation)
|
||||
if e.needStop() {
|
||||
return nil
|
||||
}
|
||||
wfCtx.IncreaseCountValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
|
||||
if err := wfCtx.Commit(); err != nil {
|
||||
return errors.WithMessage(err, "commit workflow context")
|
||||
}
|
||||
if dag {
|
||||
continue
|
||||
}
|
||||
e.checkFailedAfterRetries()
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -586,12 +627,14 @@ type engine struct {
|
||||
cli client.Client
|
||||
rk resourcekeeper.ResourceKeeper
|
||||
parentRunner string
|
||||
stepStatus map[string]common.StepStatus
|
||||
stepDependsOn map[string][]string
|
||||
}
|
||||
|
||||
func (e *engine) finishStep(operation *wfTypes.Operation) {
|
||||
if operation != nil {
|
||||
e.status.Suspend = operation.Suspend
|
||||
e.status.Terminated = operation.Terminated
|
||||
e.status.Terminated = e.status.Terminated || operation.Terminated
|
||||
}
|
||||
}
|
||||
|
||||
@@ -647,17 +690,24 @@ func (e *engine) updateStepStatus(status common.StepStatus) {
|
||||
e.status.Steps = append(e.status.Steps, common.WorkflowStepStatus{StepStatus: status})
|
||||
}
|
||||
}
|
||||
e.stepStatus[status.Name] = status
|
||||
}
|
||||
|
||||
func (e *engine) checkFailedAfterRetries() {
|
||||
if !e.waiting && e.failedAfterRetries {
|
||||
if !e.waiting && e.failedAfterRetries && feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
|
||||
e.status.Suspend = true
|
||||
}
|
||||
if e.failedAfterRetries && !feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
|
||||
e.status.Terminated = true
|
||||
}
|
||||
}
|
||||
|
||||
func (e *engine) needStop() bool {
|
||||
e.checkFailedAfterRetries()
|
||||
return e.status.Suspend || e.status.Terminated
|
||||
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
|
||||
e.checkFailedAfterRetries()
|
||||
}
|
||||
// if the workflow is terminated, we still need to execute all the remaining steps
|
||||
return e.status.Suspend
|
||||
}
|
||||
|
||||
// ComputeWorkflowRevisionHash compute workflow revision.
|
||||
@@ -678,7 +728,75 @@ func ComputeWorkflowRevisionHash(rev string, app *oamcore.Application) (string,
|
||||
|
||||
// IsFailedAfterRetry check if application is hang due to FailedAfterRetry
|
||||
func IsFailedAfterRetry(app *oamcore.Application) bool {
|
||||
return app.Status.Workflow != nil && app.Status.Workflow.Message == MessageFailedAfterRetries
|
||||
return app.Status.Workflow != nil && (app.Status.Workflow.Message == MessageTerminatedFailedAfterRetries || app.Status.Workflow.Message == MessageSuspendFailedAfterRetries)
|
||||
}
|
||||
|
||||
func (e *engine) findDependPhase(taskRunners []wfTypes.TaskRunner, index int, dag bool) common.WorkflowStepPhase {
|
||||
if e.parentRunner != "" {
|
||||
if status, ok := e.stepStatus[e.parentRunner]; ok && status.Phase == common.WorkflowStepPhaseSkipped {
|
||||
return common.WorkflowStepPhaseSkipped
|
||||
}
|
||||
}
|
||||
if dag {
|
||||
return e.findDependsOnPhase(taskRunners[index].Name())
|
||||
}
|
||||
if index < 1 {
|
||||
return common.WorkflowStepPhaseSucceeded
|
||||
}
|
||||
for i := index - 1; i >= 0; i-- {
|
||||
if e.stepStatus[taskRunners[i].Name()].Phase != common.WorkflowStepPhaseSucceeded {
|
||||
return e.stepStatus[taskRunners[i].Name()].Phase
|
||||
}
|
||||
}
|
||||
return e.stepStatus[taskRunners[index-1].Name()].Phase
|
||||
}
|
||||
|
||||
func (e *engine) findDependsOnPhase(name string) common.WorkflowStepPhase {
|
||||
for _, dependsOn := range e.stepDependsOn[name] {
|
||||
if e.stepStatus[dependsOn].Phase != common.WorkflowStepPhaseSucceeded {
|
||||
return e.stepStatus[dependsOn].Phase
|
||||
}
|
||||
if result := e.findDependsOnPhase(dependsOn); result != common.WorkflowStepPhaseSucceeded {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return common.WorkflowStepPhaseSucceeded
|
||||
}
|
||||
|
||||
func isWaitSuspendStep(step common.StepStatus) bool {
|
||||
return step.Type == wfTypes.WorkflowStepTypeSuspend && step.Phase == common.WorkflowStepPhaseRunning
|
||||
}
|
||||
|
||||
func handleBackoffTimes(wfCtx wfContext.Context, status common.StepStatus, clear bool) error {
|
||||
if clear {
|
||||
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
|
||||
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffReason, status.ID)
|
||||
} else {
|
||||
if val, exists := wfCtx.GetValueInMemory(wfTypes.ContextPrefixBackoffReason, status.ID); !exists || val != status.Message {
|
||||
wfCtx.SetValueInMemory(status.Message, wfTypes.ContextPrefixBackoffReason, status.ID)
|
||||
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
|
||||
}
|
||||
wfCtx.IncreaseCountValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
|
||||
}
|
||||
if err := wfCtx.Commit(); err != nil {
|
||||
return errors.WithMessage(err, "commit workflow context")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *engine) cleanBackoffTimesForTerminated() {
|
||||
for _, ss := range e.status.Steps {
|
||||
for _, sub := range ss.SubStepsStatus {
|
||||
if sub.Reason == custom.StatusReasonTerminate {
|
||||
e.wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, sub.ID)
|
||||
e.wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffReason, sub.ID)
|
||||
}
|
||||
}
|
||||
if ss.Reason == custom.StatusReasonTerminate {
|
||||
e.wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, ss.ID)
|
||||
e.wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffReason, ss.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *engine) GetStepStatus(stepName string) common.WorkflowStepStatus {
|
||||
|
||||
+410
-36
@@ -20,6 +20,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
@@ -29,14 +30,18 @@ import (
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
featuregatetesting "k8s.io/component-base/featuregate/testing"
|
||||
"sigs.k8s.io/yaml"
|
||||
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
|
||||
oamcore "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
|
||||
"github.com/oam-dev/kubevela/pkg/cue/model/value"
|
||||
"github.com/oam-dev/kubevela/pkg/features"
|
||||
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
|
||||
wfContext "github.com/oam-dev/kubevela/pkg/workflow/context"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/tasks"
|
||||
"github.com/oam-dev/kubevela/pkg/workflow/tasks/custom"
|
||||
wfTypes "github.com/oam-dev/kubevela/pkg/workflow/types"
|
||||
)
|
||||
|
||||
@@ -226,6 +231,170 @@ var _ = Describe("Test Workflow", func() {
|
||||
})).Should(BeEquivalentTo(""))
|
||||
})
|
||||
|
||||
It("Workflow test skipped with sub steps", func() {
|
||||
By("Test skipped with step group")
|
||||
app, runners := makeTestCase([]oamcore.WorkflowStep{
|
||||
{
|
||||
Name: "s1",
|
||||
Type: "failed-after-retries",
|
||||
},
|
||||
{
|
||||
Name: "s2",
|
||||
Type: "step-group",
|
||||
SubSteps: []common.WorkflowSubStep{
|
||||
{
|
||||
Name: "s2-sub1",
|
||||
Type: "success",
|
||||
},
|
||||
{
|
||||
Name: "s2-sub2",
|
||||
Type: "failed",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "s3",
|
||||
Type: "success",
|
||||
},
|
||||
})
|
||||
wf := NewWorkflow(app, k8sClient, common.WorkflowModeStep, false, nil)
|
||||
ctx := monitorContext.NewTraceContext(context.Background(), "test-app")
|
||||
state, err := wf.ExecuteSteps(ctx, revision, runners)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(state).Should(BeEquivalentTo(common.WorkflowStateInitializing))
|
||||
state, err = wf.ExecuteSteps(ctx, revision, runners)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(state).Should(BeEquivalentTo(common.WorkflowStateTerminated))
|
||||
app.Status.Workflow.ContextBackend = nil
|
||||
cleanStepTimeStamp(app.Status.Workflow)
|
||||
Expect(cmp.Diff(*app.Status.Workflow, common.WorkflowStatus{
|
||||
AppRevision: app.Status.Workflow.AppRevision,
|
||||
Mode: common.WorkflowModeStep,
|
||||
Terminated: true,
|
||||
Message: string(MessageTerminatedFailedAfterRetries),
|
||||
Steps: []common.WorkflowStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s1",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2",
|
||||
Type: "step-group",
|
||||
Phase: common.WorkflowStepPhaseSkipped,
|
||||
Reason: custom.StatusReasonSkip,
|
||||
},
|
||||
SubStepsStatus: []common.WorkflowSubStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2-sub1",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSkipped,
|
||||
Reason: custom.StatusReasonSkip,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2-sub2",
|
||||
Type: "failed",
|
||||
Phase: common.WorkflowStepPhaseSkipped,
|
||||
Reason: custom.StatusReasonSkip,
|
||||
},
|
||||
}},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s3",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSkipped,
|
||||
Reason: custom.StatusReasonSkip,
|
||||
},
|
||||
}},
|
||||
})).Should(BeEquivalentTo(""))
|
||||
})
|
||||
|
||||
It("Workflow test if-always with sub steps", func() {
|
||||
By("Test if-always with step group")
|
||||
app, runners := makeTestCase([]oamcore.WorkflowStep{
|
||||
{
|
||||
Name: "s1",
|
||||
Type: "failed-after-retries",
|
||||
},
|
||||
{
|
||||
Name: "s2",
|
||||
If: "always",
|
||||
Type: "step-group",
|
||||
SubSteps: []common.WorkflowSubStep{
|
||||
{
|
||||
Name: "s2-sub1",
|
||||
DependsOn: []string{"s2-sub2"},
|
||||
If: "always",
|
||||
Type: "success",
|
||||
},
|
||||
{
|
||||
Name: "s2-sub2",
|
||||
Type: "failed-after-retries",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "s3",
|
||||
Type: "success",
|
||||
},
|
||||
})
|
||||
wf := NewWorkflow(app, k8sClient, common.WorkflowModeStep, false, nil)
|
||||
ctx := monitorContext.NewTraceContext(context.Background(), "test-app")
|
||||
state, err := wf.ExecuteSteps(ctx, revision, runners)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(state).Should(BeEquivalentTo(common.WorkflowStateInitializing))
|
||||
state, err = wf.ExecuteSteps(ctx, revision, runners)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(state).Should(BeEquivalentTo(common.WorkflowStateTerminated))
|
||||
app.Status.Workflow.ContextBackend = nil
|
||||
cleanStepTimeStamp(app.Status.Workflow)
|
||||
Expect(cmp.Diff(*app.Status.Workflow, common.WorkflowStatus{
|
||||
AppRevision: app.Status.Workflow.AppRevision,
|
||||
Mode: common.WorkflowModeStep,
|
||||
Terminated: true,
|
||||
Message: string(MessageTerminatedFailedAfterRetries),
|
||||
Steps: []common.WorkflowStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s1",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2",
|
||||
Type: "step-group",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
SubStepsStatus: []common.WorkflowSubStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2-sub1",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2-sub2",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
}},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s3",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSkipped,
|
||||
Reason: custom.StatusReasonSkip,
|
||||
},
|
||||
}},
|
||||
})).Should(BeEquivalentTo(""))
|
||||
})
|
||||
|
||||
It("Workflow test success with sub steps", func() {
|
||||
By("Test success with step group")
|
||||
app, runners := makeTestCase([]oamcore.WorkflowStep{
|
||||
@@ -352,8 +521,9 @@ var _ = Describe("Test Workflow", func() {
|
||||
})).Should(BeEquivalentTo(""))
|
||||
})
|
||||
|
||||
It("Workflow test for failed after retries", func() {
|
||||
By("Test failed-after-retries in StepByStep mode")
|
||||
It("Workflow test for failed after retries with suspend", func() {
|
||||
By("Test failed-after-retries in StepByStep mode with suspend")
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
|
||||
app, runners := makeTestCase([]oamcore.WorkflowStep{
|
||||
{
|
||||
Name: "s1",
|
||||
@@ -384,7 +554,7 @@ var _ = Describe("Test Workflow", func() {
|
||||
Expect(cmp.Diff(*workflowStatus, common.WorkflowStatus{
|
||||
AppRevision: workflowStatus.AppRevision,
|
||||
Mode: common.WorkflowModeStep,
|
||||
Message: MessageFailedAfterRetries,
|
||||
Message: MessageSuspendFailedAfterRetries,
|
||||
Suspend: true,
|
||||
Steps: []common.WorkflowStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
@@ -394,14 +564,15 @@ var _ = Describe("Test Workflow", func() {
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Name: "s2",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
}},
|
||||
})).Should(BeEquivalentTo(""))
|
||||
|
||||
By("Test failed-after-retries in DAG mode")
|
||||
By("Test failed-after-retries in DAG mode with suspend")
|
||||
app, runners = makeTestCase([]oamcore.WorkflowStep{
|
||||
{
|
||||
Name: "s1",
|
||||
@@ -432,7 +603,7 @@ var _ = Describe("Test Workflow", func() {
|
||||
Expect(cmp.Diff(*workflowStatus, common.WorkflowStatus{
|
||||
AppRevision: workflowStatus.AppRevision,
|
||||
Mode: common.WorkflowModeDAG,
|
||||
Message: MessageFailedAfterRetries,
|
||||
Message: MessageSuspendFailedAfterRetries,
|
||||
Suspend: true,
|
||||
Steps: []common.WorkflowStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
@@ -442,9 +613,10 @@ var _ = Describe("Test Workflow", func() {
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Name: "s2",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
@@ -456,8 +628,183 @@ var _ = Describe("Test Workflow", func() {
|
||||
})).Should(BeEquivalentTo(""))
|
||||
})
|
||||
|
||||
It("Workflow test if always", func() {
|
||||
By("Test if always in StepByStep mode")
|
||||
app, runners := makeTestCase([]oamcore.WorkflowStep{
|
||||
{
|
||||
Name: "s1",
|
||||
Type: "success",
|
||||
},
|
||||
{
|
||||
Name: "s2",
|
||||
Type: "failed-after-retries",
|
||||
},
|
||||
{
|
||||
Name: "s3",
|
||||
If: "always",
|
||||
Type: "success",
|
||||
},
|
||||
{
|
||||
Name: "s4",
|
||||
Type: "success",
|
||||
},
|
||||
{
|
||||
Name: "s5",
|
||||
If: "always",
|
||||
Type: "success",
|
||||
},
|
||||
})
|
||||
ctx := monitorContext.NewTraceContext(context.Background(), "test-app")
|
||||
wf := NewWorkflow(app, k8sClient, common.WorkflowModeStep, false, nil)
|
||||
state, err := wf.ExecuteSteps(ctx, revision, runners)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(state).Should(BeEquivalentTo(common.WorkflowStateInitializing))
|
||||
|
||||
state, err = wf.ExecuteSteps(ctx, revision, runners)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(state).Should(BeEquivalentTo(common.WorkflowStateTerminated))
|
||||
workflowStatus := app.Status.Workflow
|
||||
Expect(workflowStatus.ContextBackend.Name).Should(BeEquivalentTo("workflow-" + app.Name + "-context"))
|
||||
workflowStatus.ContextBackend = nil
|
||||
cleanStepTimeStamp(workflowStatus)
|
||||
Expect(cmp.Diff(*workflowStatus, common.WorkflowStatus{
|
||||
AppRevision: workflowStatus.AppRevision,
|
||||
Mode: common.WorkflowModeStep,
|
||||
Message: MessageTerminatedFailedAfterRetries,
|
||||
Suspend: false,
|
||||
Terminated: true,
|
||||
Steps: []common.WorkflowStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s1",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s3",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s4",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSkipped,
|
||||
Reason: custom.StatusReasonSkip,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s5",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}},
|
||||
})).Should(BeEquivalentTo(""))
|
||||
|
||||
By("Test if always in DAG mode")
|
||||
app, runners = makeTestCase([]oamcore.WorkflowStep{
|
||||
{
|
||||
Name: "s1",
|
||||
Type: "success",
|
||||
},
|
||||
{
|
||||
Name: "s2",
|
||||
Type: "failed-after-retries",
|
||||
},
|
||||
{
|
||||
Name: "s3",
|
||||
DependsOn: []string{"s2"},
|
||||
If: "always",
|
||||
Type: "success",
|
||||
},
|
||||
{
|
||||
Name: "s4",
|
||||
DependsOn: []string{"s3"},
|
||||
Type: "success",
|
||||
},
|
||||
{
|
||||
Name: "s5",
|
||||
DependsOn: []string{"s3"},
|
||||
If: "always",
|
||||
Type: "success",
|
||||
},
|
||||
{
|
||||
Name: "s6",
|
||||
DependsOn: []string{"s1", "s5"},
|
||||
Type: "success",
|
||||
},
|
||||
})
|
||||
ctx = monitorContext.NewTraceContext(context.Background(), "test-app")
|
||||
wf = NewWorkflow(app, k8sClient, common.WorkflowModeDAG, false, nil)
|
||||
state, err = wf.ExecuteSteps(ctx, revision, runners)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(state).Should(BeEquivalentTo(common.WorkflowStateInitializing))
|
||||
|
||||
state, err = wf.ExecuteSteps(ctx, revision, runners)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(state).Should(BeEquivalentTo(common.WorkflowStateTerminated))
|
||||
workflowStatus = app.Status.Workflow
|
||||
Expect(workflowStatus.ContextBackend.Name).Should(BeEquivalentTo("workflow-" + app.Name + "-context"))
|
||||
workflowStatus.ContextBackend = nil
|
||||
cleanStepTimeStamp(workflowStatus)
|
||||
Expect(cmp.Diff(*workflowStatus, common.WorkflowStatus{
|
||||
AppRevision: workflowStatus.AppRevision,
|
||||
Mode: common.WorkflowModeDAG,
|
||||
Message: MessageTerminatedFailedAfterRetries,
|
||||
Terminated: true,
|
||||
Steps: []common.WorkflowStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s1",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s3",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s4",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSkipped,
|
||||
Reason: custom.StatusReasonSkip,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s5",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSucceeded,
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s6",
|
||||
Type: "success",
|
||||
Phase: common.WorkflowStepPhaseSkipped,
|
||||
Reason: custom.StatusReasonSkip,
|
||||
},
|
||||
}},
|
||||
})).Should(BeEquivalentTo(""))
|
||||
})
|
||||
|
||||
It("Test failed after retries with sub steps", func() {
|
||||
By("Test failed-after-retries with step group in StepByStep mode")
|
||||
defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
|
||||
app, runners := makeTestCase([]oamcore.WorkflowStep{
|
||||
{
|
||||
Name: "s1",
|
||||
@@ -495,7 +842,7 @@ var _ = Describe("Test Workflow", func() {
|
||||
Expect(cmp.Diff(*app.Status.Workflow, common.WorkflowStatus{
|
||||
AppRevision: app.Status.Workflow.AppRevision,
|
||||
Mode: common.WorkflowModeStep,
|
||||
Message: MessageFailedAfterRetries,
|
||||
Message: MessageSuspendFailedAfterRetries,
|
||||
Suspend: true,
|
||||
Steps: []common.WorkflowStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
@@ -505,9 +852,10 @@ var _ = Describe("Test Workflow", func() {
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2",
|
||||
Type: "step-group",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Name: "s2",
|
||||
Type: "step-group",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
SubStepsStatus: []common.WorkflowSubStepStatus{{
|
||||
StepStatus: common.StepStatus{
|
||||
@@ -517,9 +865,10 @@ var _ = Describe("Test Workflow", func() {
|
||||
},
|
||||
}, {
|
||||
StepStatus: common.StepStatus{
|
||||
Name: "s2-sub2",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Name: "s2-sub2",
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
},
|
||||
}},
|
||||
}},
|
||||
@@ -1031,11 +1380,11 @@ func makeTestCase(steps []oamcore.WorkflowStep) (*oamcore.Application, []wfTypes
|
||||
if step.SubSteps != nil {
|
||||
subStepRunners := []wfTypes.TaskRunner{}
|
||||
for _, subStep := range step.SubSteps {
|
||||
subStepRunners = append(subStepRunners, makeRunner(subStep.Name, subStep.Type, nil))
|
||||
subStepRunners = append(subStepRunners, makeRunner(subStep.Name, subStep.Type, subStep.If, subStep.DependsOn, nil))
|
||||
}
|
||||
runners = append(runners, makeRunner(step.Name, step.Type, subStepRunners))
|
||||
runners = append(runners, makeRunner(step.Name, step.Type, step.If, step.DependsOn, subStepRunners))
|
||||
} else {
|
||||
runners = append(runners, makeRunner(step.Name, step.Type, nil))
|
||||
runners = append(runners, makeRunner(step.Name, step.Type, step.If, step.DependsOn, nil))
|
||||
}
|
||||
}
|
||||
return app, runners
|
||||
@@ -1043,8 +1392,26 @@ func makeTestCase(steps []oamcore.WorkflowStep) (*oamcore.Application, []wfTypes
|
||||
|
||||
var pending bool
|
||||
|
||||
func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wfTypes.TaskRunner {
|
||||
func makeRunner(name, tpy, ifDecl string, dependsOn []string, subTaskRunners []wfTypes.TaskRunner) wfTypes.TaskRunner {
|
||||
var run func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error)
|
||||
skip := func(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
|
||||
status := common.StepStatus{
|
||||
Name: name,
|
||||
Type: tpy,
|
||||
}
|
||||
if utilfeature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
|
||||
return status, false
|
||||
}
|
||||
skip := custom.SkipTaskRunner(&custom.SkipOptions{
|
||||
If: ifDecl,
|
||||
DependsOnPhase: dependsOnPhase,
|
||||
})
|
||||
if skip {
|
||||
status.Phase = common.WorkflowStepPhaseSkipped
|
||||
status.Reason = custom.StatusReasonSkip
|
||||
}
|
||||
return status, skip
|
||||
}
|
||||
switch tpy {
|
||||
case "suspend":
|
||||
run = func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error) {
|
||||
@@ -1085,9 +1452,10 @@ func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wf
|
||||
case "failed-after-retries":
|
||||
run = func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error) {
|
||||
return common.StepStatus{
|
||||
Name: name,
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Name: name,
|
||||
Type: "failed-after-retries",
|
||||
Phase: common.WorkflowStepPhaseFailed,
|
||||
Reason: custom.StatusReasonFailedAfterRetries,
|
||||
}, &wfTypes.Operation{
|
||||
FailedAfterRetries: true,
|
||||
}, nil
|
||||
@@ -1111,8 +1479,14 @@ func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wf
|
||||
}, &wfTypes.Operation{}, err
|
||||
}
|
||||
case "step-group":
|
||||
group, _ := tasks.StepGroup(oamcore.WorkflowStep{Name: name}, &wfTypes.GeneratorOptions{SubTaskRunners: subTaskRunners})
|
||||
group, _ := tasks.StepGroup(oamcore.WorkflowStep{
|
||||
Name: name,
|
||||
If: ifDecl,
|
||||
Type: tpy,
|
||||
DependsOn: dependsOn,
|
||||
}, &wfTypes.GeneratorOptions{SubTaskRunners: subTaskRunners})
|
||||
run = group.Run
|
||||
skip = group.Skip
|
||||
default:
|
||||
run = func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error) {
|
||||
return common.StepStatus{
|
||||
@@ -1126,7 +1500,7 @@ func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wf
|
||||
return &testTaskRunner{
|
||||
name: name,
|
||||
run: run,
|
||||
checkPending: func(ctx wfContext.Context) bool {
|
||||
checkPending: func(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
|
||||
if tpy != "pending" {
|
||||
return false
|
||||
}
|
||||
@@ -1135,7 +1509,7 @@ func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wf
|
||||
}
|
||||
return false
|
||||
},
|
||||
subTaskRunners: subTaskRunners,
|
||||
skip: skip,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1156,10 +1530,10 @@ metadata:
|
||||
)
|
||||
|
||||
type testTaskRunner struct {
|
||||
name string
|
||||
run func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error)
|
||||
checkPending func(ctx wfContext.Context) bool
|
||||
subTaskRunners []wfTypes.TaskRunner
|
||||
name string
|
||||
run func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error)
|
||||
checkPending func(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool
|
||||
skip func(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool)
|
||||
}
|
||||
|
||||
// Name return step name.
|
||||
@@ -1173,12 +1547,12 @@ func (tr *testTaskRunner) Run(ctx wfContext.Context, options *wfTypes.TaskRunOpt
|
||||
}
|
||||
|
||||
// Pending check task should be executed or not.
|
||||
func (tr *testTaskRunner) Pending(ctx wfContext.Context) bool {
|
||||
return tr.checkPending(ctx)
|
||||
func (tr *testTaskRunner) Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
|
||||
return tr.checkPending(ctx, stepStatus)
|
||||
}
|
||||
|
||||
func (tr *testTaskRunner) SubTaskRunners() []wfTypes.TaskRunner {
|
||||
return tr.subTaskRunners
|
||||
func (tr *testTaskRunner) Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
|
||||
return tr.skip(dependsOnPhase, stepStatus)
|
||||
}
|
||||
|
||||
func cleanStepTimeStamp(wfStatus *common.WorkflowStatus) {
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
oamcommon "github.com/oam-dev/kubevela/apis/core.oam.dev/common"
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
|
||||
"github.com/oam-dev/kubevela/apis/types"
|
||||
"github.com/oam-dev/kubevela/pkg/apiserver/domain/service"
|
||||
"github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha2/application"
|
||||
"github.com/oam-dev/kubevela/pkg/controller/utils"
|
||||
"github.com/oam-dev/kubevela/pkg/oam"
|
||||
@@ -170,7 +171,7 @@ func NewWorkflowTerminateCommand(c common.Args, ioStream cmdutil.IOStreams) *cob
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = terminateWorkflow(client, app)
|
||||
err = TerminateWorkflow(client, app)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -286,11 +287,9 @@ func resumeWorkflow(kubecli client.Client, app *v1beta1.Application) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func terminateWorkflow(kubecli client.Client, app *v1beta1.Application) error {
|
||||
// set the workflow terminated to true
|
||||
app.Status.Workflow.Terminated = true
|
||||
|
||||
if err := kubecli.Status().Patch(context.TODO(), app, client.Merge); err != nil {
|
||||
// TerminateWorkflow terminate workflow
|
||||
func TerminateWorkflow(kubecli client.Client, app *v1beta1.Application) error {
|
||||
if err := service.TerminateWorkflow(context.TODO(), kubecli, app); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -122,15 +122,15 @@ var _ = Describe("Application Normal tests", func() {
|
||||
}, time.Second*5, time.Millisecond*500).Should(Succeed())
|
||||
}
|
||||
|
||||
verifyApplicationWorkflowSuspending := func(ns, appName string) {
|
||||
verifyApplicationWorkflowTerminated := func(ns, appName string) {
|
||||
var testApp v1beta1.Application
|
||||
Eventually(func() error {
|
||||
err := k8sClient.Get(ctx, client.ObjectKey{Namespace: ns, Name: appName}, &testApp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if testApp.Status.Phase != oamcomm.ApplicationWorkflowSuspending {
|
||||
return fmt.Errorf("application status wants %s, actually %s", oamcomm.ApplicationWorkflowSuspending, testApp.Status.Phase)
|
||||
if testApp.Status.Phase != oamcomm.ApplicationWorkflowTerminated {
|
||||
return fmt.Errorf("application status wants %s, actually %s", oamcomm.ApplicationWorkflowTerminated, testApp.Status.Phase)
|
||||
}
|
||||
return nil
|
||||
}, 120*time.Second, time.Second).Should(BeNil())
|
||||
@@ -335,7 +335,7 @@ var _ = Describe("Application Normal tests", func() {
|
||||
Expect(k8sClient.Create(ctx, &newApp)).Should(BeNil())
|
||||
|
||||
By("check application status")
|
||||
verifyApplicationWorkflowSuspending(newApp.Namespace, newApp.Name)
|
||||
verifyApplicationWorkflowTerminated(newApp.Namespace, newApp.Name)
|
||||
})
|
||||
|
||||
It("Test wait suspend", func() {
|
||||
@@ -425,7 +425,7 @@ var _ = Describe("Application Normal tests", func() {
|
||||
Expect(k8sClient.Create(ctx, &newApp)).Should(BeNil())
|
||||
|
||||
By("Checking an application status")
|
||||
verifyApplicationWorkflowSuspending(newApp.Namespace, newApp.Name)
|
||||
verifyApplicationWorkflowTerminated(newApp.Namespace, newApp.Name)
|
||||
})
|
||||
|
||||
It("Test app with non-existence ServiceAccount", func() {
|
||||
@@ -453,6 +453,6 @@ var _ = Describe("Application Normal tests", func() {
|
||||
Expect(k8sClient.Create(ctx, &newApp)).Should(BeNil())
|
||||
|
||||
By("Checking an application status")
|
||||
verifyApplicationWorkflowSuspending(newApp.Namespace, newApp.Name)
|
||||
verifyApplicationWorkflowTerminated(newApp.Namespace, newApp.Name)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user