Feat: add if in workflow (#3941)

* Feat: add if in workflow struct

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Feat: implement the if in workflow

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Feat: support dependency and skip for suspend step

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Fix: fix the rebase from sub steps

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Fix: fix the lint

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Feat: support if in sub steps

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Feat: add tests in application controller

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Fix: fix the lint

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Test: add more tests in discover and custom

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Lint: fix lint

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Tests: add more tests in application controller

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Fix: change failed after retries into reason

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* Fix: fix the terminate cli

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* fix lint

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* remove the terminate workflow to pkg and add feature gates

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* resolve comments

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* nit fix

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>

* make finish condition more clear

Signed-off-by: FogDong <dongtianxin.tx@alibaba-inc.com>
This commit is contained in:
Tianxin Dong
2022-05-27 22:01:14 +08:00
committed by GitHub
parent fd024bc3e2
commit fcfb1012d6
31 changed files with 1442 additions and 273 deletions
+6
View File
@@ -347,6 +347,8 @@ type WorkflowStep struct {
SubSteps []WorkflowSubStep `json:"subSteps,omitempty"`
If string `json:"if,omitempty"`
DependsOn []string `json:"dependsOn,omitempty"`
Inputs StepInputs `json:"inputs,omitempty"`
@@ -364,6 +366,8 @@ type WorkflowSubStep struct {
// +kubebuilder:pruning:PreserveUnknownFields
Properties *runtime.RawExtension `json:"properties,omitempty"`
If string `json:"if,omitempty"`
DependsOn []string `json:"dependsOn,omitempty"`
Inputs StepInputs `json:"inputs,omitempty"`
@@ -397,6 +401,8 @@ const (
WorkflowStepPhaseSucceeded WorkflowStepPhase = "succeeded"
// WorkflowStepPhaseFailed will report error in `message`.
WorkflowStepPhaseFailed WorkflowStepPhase = "failed"
// WorkflowStepPhaseSkipped will make the controller skip the step.
WorkflowStepPhaseSkipped WorkflowStepPhase = "skipped"
// WorkflowStepPhaseStopped will make the controller stop the workflow.
WorkflowStepPhaseStopped WorkflowStepPhase = "stopped"
// WorkflowStepPhaseRunning will make the controller continue the workflow.
+6 -5
View File
@@ -53,11 +53,12 @@ helm install --create-namespace -n vela-system kubevela kubevela/vela-core --wai
### KubeVela workflow parameters
| Name | Description | Value |
| -------------------------------------- | ------------------------------------------------------ | ----- |
| `workflow.backoff.maxTime.waitState` | The max backoff time of workflow in a wait condition | `60` |
| `workflow.backoff.maxTime.failedState` | The max backoff time of workflow in a failed condition | `300` |
| `workflow.step.errorRetryTimes` | The max retry times of a failed workflow step | `10` |
| Name | Description | Value |
| -------------------------------------- | ------------------------------------------------------ | ------- |
| `workflow.enableSuspendOnFailure` | Enable suspend on workflow failure | `false` |
| `workflow.backoff.maxTime.waitState` | The max backoff time of workflow in a wait condition | `60` |
| `workflow.backoff.maxTime.failedState` | The max backoff time of workflow in a failed condition | `300` |
| `workflow.step.errorRetryTimes` | The max retry times of a failed workflow step | `10` |
### KubeVela controller parameters
@@ -2209,6 +2209,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of
WorkflowStep
@@ -2253,6 +2255,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input
of WorkflowStep
@@ -3954,6 +3958,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -3995,6 +4001,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of
WorkflowStep
@@ -1020,6 +1020,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -1061,6 +1063,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of
WorkflowStep
@@ -42,6 +42,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -83,6 +85,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -147,6 +151,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -188,6 +194,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -172,6 +172,7 @@ spec:
- "--max-workflow-wait-backoff-time={{ .Values.workflow.backoff.maxTime.waitState }}"
- "--max-workflow-failed-backoff-time={{ .Values.workflow.backoff.maxTime.failedState }}"
- "--max-workflow-step-error-retry-times={{ .Values.workflow.step.errorRetryTimes }}"
- "--feature-gates=EnableSuspendOnFailure={{- .Values.workflow.enableSuspendOnFailure | toString -}}"
- "--feature-gates=AuthenticateApplication={{- .Values.authentication.enabled | toString -}}"
{{ if .Values.authentication.enabled }}
{{ if .Values.authentication.withUser }}
+2
View File
@@ -35,10 +35,12 @@ dependCheckWait: 30s
## @section KubeVela workflow parameters
## @param workflow.enableSuspendOnFailure Enable suspend on workflow failure
## @param workflow.backoff.maxTime.waitState The max backoff time of workflow in a wait condition
## @param workflow.backoff.maxTime.failedState The max backoff time of workflow in a failed condition
## @param workflow.step.errorRetryTimes The max retry times of a failed workflow step
workflow:
enableSuspendOnFailure: false
backoff:
maxTime:
waitState: 60
+6 -5
View File
@@ -72,11 +72,12 @@ helm install --create-namespace -n vela-system kubevela kubevela/vela-minimal --
### KubeVela workflow parameters
| Name | Description | Value |
| -------------------------------------- | ------------------------------------------------------ | ----- |
| `workflow.backoff.maxTime.waitState` | The max backoff time of workflow in a wait condition | `60` |
| `workflow.backoff.maxTime.failedState` | The max backoff time of workflow in a failed condition | `300` |
| `workflow.step.errorRetryTimes` | The max retry times of a failed workflow step | `10` |
| Name | Description | Value |
| -------------------------------------- | ------------------------------------------------------ | ------- |
| `workflow.enableSuspendOnFailure` | Enable suspend on workflow failure | `false` |
| `workflow.backoff.maxTime.waitState` | The max backoff time of workflow in a wait condition | `60` |
| `workflow.backoff.maxTime.failedState` | The max backoff time of workflow in a failed condition | `300` |
| `workflow.step.errorRetryTimes` | The max retry times of a failed workflow step | `10` |
### KubeVela controller parameters
@@ -2209,6 +2209,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of
WorkflowStep
@@ -2253,6 +2255,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input
of WorkflowStep
@@ -3954,6 +3958,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -3995,6 +4001,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of
WorkflowStep
@@ -1020,6 +1020,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -1061,6 +1063,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of
WorkflowStep
@@ -142,6 +142,7 @@ spec:
- "--max-workflow-wait-backoff-time={{ .Values.workflow.backoff.maxTime.waitState }}"
- "--max-workflow-failed-backoff-time={{ .Values.workflow.backoff.maxTime.failedState }}"
- "--max-workflow-step-error-retry-times={{ .Values.workflow.step.errorRetryTimes }}"
- "--feature-gates=EnableSuspendOnFailure={{- .Values.workflow.enableSuspendOnFailure | toString -}}"
- "--feature-gates=AuthenticateApplication={{- .Values.authentication.enabled | toString -}}"
{{ if .Values.authentication.enabled }}
{{ if .Values.authentication.withUser }}
+2
View File
@@ -38,10 +38,12 @@ dependCheckWait: 30s
## @section KubeVela workflow parameters
## @param workflow.enableSuspendOnFailure Enable suspend on workflow failure
## @param workflow.backoff.maxTime.waitState The max backoff time of workflow in a wait condition
## @param workflow.backoff.maxTime.failedState The max backoff time of workflow in a failed condition
## @param workflow.step.errorRetryTimes The max retry times of a failed workflow step
workflow:
enableSuspendOnFailure: false
backoff:
maxTime:
waitState: 60
@@ -2209,6 +2209,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of
WorkflowStep
@@ -2253,6 +2255,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input
of WorkflowStep
@@ -3954,6 +3958,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -3995,6 +4001,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of
WorkflowStep
@@ -1021,6 +1021,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -1062,6 +1064,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of
WorkflowStep
@@ -42,6 +42,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -83,6 +85,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -147,6 +151,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
@@ -188,6 +194,8 @@ spec:
items:
type: string
type: array
if:
type: string
inputs:
description: StepInputs defines variable input of WorkflowStep
items:
+38 -2
View File
@@ -43,6 +43,7 @@ import (
"github.com/oam-dev/kubevela/pkg/oam/util"
utils2 "github.com/oam-dev/kubevela/pkg/utils"
"github.com/oam-dev/kubevela/pkg/utils/apply"
"github.com/oam-dev/kubevela/pkg/workflow/tasks/custom"
)
// WorkflowService workflow manage api
@@ -577,8 +578,7 @@ func (w *workflowServiceImpl) TerminateRecord(ctx context.Context, appModel *mod
return err
}
oamApp.Status.Workflow.Terminated = true
if err := w.KubeClient.Status().Patch(ctx, oamApp, client.Merge); err != nil {
if err := TerminateWorkflow(ctx, w.KubeClient, oamApp); err != nil {
return err
}
if err := w.syncWorkflowStatus(ctx, oamApp, recordName, oamApp.Name); err != nil {
@@ -588,6 +588,42 @@ func (w *workflowServiceImpl) TerminateRecord(ctx context.Context, appModel *mod
return nil
}
// TerminateWorkflow terminate workflow
func TerminateWorkflow(ctx context.Context, kubecli client.Client, app *v1beta1.Application) error {
// set the workflow terminated to true
app.Status.Workflow.Terminated = true
steps := app.Status.Workflow.Steps
for i, step := range steps {
switch step.Phase {
case common.WorkflowStepPhaseFailed:
if step.Reason != custom.StatusReasonFailedAfterRetries {
steps[i].Reason = custom.StatusReasonTerminate
}
case common.WorkflowStepPhaseRunning:
steps[i].Phase = common.WorkflowStepPhaseFailed
steps[i].Reason = custom.StatusReasonTerminate
default:
}
for j, sub := range step.SubStepsStatus {
switch sub.Phase {
case common.WorkflowStepPhaseFailed:
if sub.Reason != custom.StatusReasonFailedAfterRetries {
steps[i].SubStepsStatus[j].Phase = custom.StatusReasonTerminate
}
case common.WorkflowStepPhaseRunning:
steps[i].SubStepsStatus[j].Phase = common.WorkflowStepPhaseFailed
steps[i].SubStepsStatus[j].Reason = custom.StatusReasonTerminate
default:
}
}
}
if err := kubecli.Status().Patch(ctx, app, client.Merge); err != nil {
return err
}
return nil
}
func (w *workflowServiceImpl) RollbackRecord(ctx context.Context, appModel *model.Application, workflow *model.Workflow, recordName, revisionVersion string) error {
if revisionVersion == "" {
// find the latest complete revision version
@@ -29,6 +29,7 @@ import (
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/util/workqueue"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -47,6 +48,7 @@ import (
common2 "github.com/oam-dev/kubevela/pkg/controller/common"
core "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev"
"github.com/oam-dev/kubevela/pkg/cue/packages"
"github.com/oam-dev/kubevela/pkg/features"
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
"github.com/oam-dev/kubevela/pkg/monitor/metrics"
"github.com/oam-dev/kubevela/pkg/oam"
@@ -229,7 +231,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
handler.app.Status.Workflow.SuspendState = ""
return r.gcResourceTrackers(logCtx, handler, common.ApplicationRunningWorkflow, false, false)
}
if !workflow.IsFailedAfterRetry(app) {
if !workflow.IsFailedAfterRetry(app) || !feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
r.stateKeep(logCtx, handler, app)
}
return r.gcResourceTrackers(logCtx, handler, common.ApplicationWorkflowSuspending, false, true)
@@ -23,8 +23,11 @@ import (
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/utils/pointer"
. "github.com/onsi/ginkgo"
@@ -50,6 +53,7 @@ import (
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
stdv1alpha1 "github.com/oam-dev/kubevela/apis/standard.oam.dev/v1alpha1"
velatypes "github.com/oam-dev/kubevela/apis/types"
"github.com/oam-dev/kubevela/pkg/features"
"github.com/oam-dev/kubevela/pkg/oam"
"github.com/oam-dev/kubevela/pkg/oam/testutil"
"github.com/oam-dev/kubevela/pkg/oam/util"
@@ -1896,6 +1900,7 @@ var _ = Describe("Test Application Controller", func() {
})
It("application with dag workflow failed after retries", func() {
defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
ns := corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "dag-failed-after-retries",
@@ -1938,7 +1943,7 @@ var _ = Describe("Test Application Controller", func() {
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunningWorkflow))
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageInitializingWorkflow))
By("verify the first twenty reconciles")
By("verify the first ten reconciles")
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes; i++ {
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
@@ -1947,12 +1952,13 @@ var _ = Describe("Test Application Controller", func() {
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
}
By("application should be suspended after failed twenty reconciles")
By("application should be suspended after failed max reconciles")
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationWorkflowSuspending))
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageFailedAfterRetries))
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageSuspendFailedAfterRetries))
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
Expect(checkApp.Status.Workflow.Steps[1].Reason).Should(BeEquivalentTo(custom.StatusReasonFailedAfterRetries))
By("resume the suspended application")
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
@@ -1984,7 +1990,7 @@ var _ = Describe("Test Application Controller", func() {
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunningWorkflow))
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes+1; i++ {
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes-1; i++ {
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunningWorkflow))
@@ -1992,9 +1998,18 @@ var _ = Describe("Test Application Controller", func() {
Expect(checkApp.Status.Workflow.Steps[0].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseRunning))
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
}
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunningWorkflow))
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(string(common.WorkflowStateExecuting)))
Expect(checkApp.Status.Workflow.Steps[0].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseRunning))
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
Expect(checkApp.Status.Workflow.Steps[1].Reason).Should(BeEquivalentTo(custom.StatusReasonFailedAfterRetries))
})
It("application with step by step workflow failed after retries", func() {
defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
ns := corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "step-by-step-failed-after-retries",
@@ -2060,12 +2075,13 @@ var _ = Describe("Test Application Controller", func() {
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
}
By("application should be suspended after failed twenty reconciles")
By("application should be suspended after failed max reconciles")
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationWorkflowSuspending))
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageFailedAfterRetries))
Expect(checkApp.Status.Workflow.Message).Should(BeEquivalentTo(workflow.MessageSuspendFailedAfterRetries))
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
Expect(checkApp.Status.Workflow.Steps[1].Reason).Should(BeEquivalentTo(custom.StatusReasonFailedAfterRetries))
By("resume the suspended application")
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
@@ -2078,6 +2094,352 @@ var _ = Describe("Test Application Controller", func() {
Expect(checkApp.Status.Workflow.Steps[1].Phase).Should(BeEquivalentTo(common.WorkflowStepPhaseFailed))
})
It("application with sub steps", func() {
ns := corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "app-with-sub-steps",
},
}
Expect(k8sClient.Create(ctx, &ns)).Should(BeNil())
healthComponentDef := &v1beta1.ComponentDefinition{}
hCDefJson, _ := yaml.YAMLToJSON([]byte(cdDefWithHealthStatusYaml))
Expect(json.Unmarshal(hCDefJson, healthComponentDef)).Should(BeNil())
healthComponentDef.Name = "worker-with-health"
healthComponentDef.Namespace = "app-with-sub-steps"
Expect(k8sClient.Create(ctx, healthComponentDef)).Should(BeNil())
app := &v1beta1.Application{
TypeMeta: metav1.TypeMeta{
Kind: "Application",
APIVersion: "core.oam.dev/v1beta1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "app-with-sub-steps",
Namespace: "app-with-sub-steps",
},
Spec: v1beta1.ApplicationSpec{
Components: []common.ApplicationComponent{
{
Name: "myweb1",
Type: "worker-with-health",
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
},
{
Name: "myweb2",
Type: "worker",
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
},
{
Name: "myweb3",
Type: "worker-with-health",
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
},
},
Workflow: &v1beta1.Workflow{
Steps: []v1beta1.WorkflowStep{
{
Name: "myweb1",
Type: "apply-component",
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb1"}`)},
},
{
Name: "myweb2",
Type: "step-group",
SubSteps: []common.WorkflowSubStep{
{
Name: "myweb2-sub1",
Type: "apply-component",
DependsOn: []string{"myweb2-sub2"},
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb2"}`)},
},
{
Name: "myweb2-sub2",
Type: "apply-component",
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb3"}`)},
},
},
},
},
},
},
}
Expect(k8sClient.Create(context.Background(), app)).Should(BeNil())
appKey := types.NamespacedName{Namespace: ns.Name, Name: app.Name}
testutil.ReconcileOnceAfterFinalizer(reconciler, reconcile.Request{NamespacedName: appKey})
expDeployment := &v1.Deployment{}
web2Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb2"}
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
web3Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb3"}
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(util.NotFoundMatcher{})
checkApp := &v1beta1.Application{}
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
web1Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb1"}
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(BeNil())
expDeployment.Status.Replicas = 1
expDeployment.Status.ReadyReplicas = 1
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(BeNil())
expDeployment.Status.Replicas = 1
expDeployment.Status.ReadyReplicas = 1
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(BeNil())
expDeployment.Status.Replicas = 1
expDeployment.Status.ReadyReplicas = 1
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
checkApp = &v1beta1.Application{}
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationRunning))
})
It("application with if always in workflow", func() {
ns := corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "app-with-if-always-workflow",
},
}
Expect(k8sClient.Create(ctx, &ns)).Should(BeNil())
healthComponentDef := &v1beta1.ComponentDefinition{}
hCDefJson, _ := yaml.YAMLToJSON([]byte(cdDefWithHealthStatusYaml))
Expect(json.Unmarshal(hCDefJson, healthComponentDef)).Should(BeNil())
healthComponentDef.Name = "worker-with-health"
healthComponentDef.Namespace = "app-with-if-always-workflow"
Expect(k8sClient.Create(ctx, healthComponentDef)).Should(BeNil())
app := &v1beta1.Application{
TypeMeta: metav1.TypeMeta{
Kind: "Application",
APIVersion: "core.oam.dev/v1beta1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "app-with-if-always-workflow",
Namespace: "app-with-if-always-workflow",
},
Spec: v1beta1.ApplicationSpec{
Components: []common.ApplicationComponent{
{
Name: "myweb1",
Type: "worker-with-health",
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
},
{
Name: "myweb2",
Type: "worker",
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
},
{
Name: "failed-step",
Type: "k8s-objects",
Properties: &runtime.RawExtension{Raw: []byte(`{"objects":[{"apiVersion":"v1","kind":"invalid","metadata":{"name":"test1"}}]}`)},
},
},
Workflow: &v1beta1.Workflow{
Steps: []v1beta1.WorkflowStep{
{
Name: "failed-step",
Type: "apply-component",
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"failed-step"}`)},
},
{
Name: "myweb1",
Type: "apply-component",
If: "always",
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb1"}`)},
},
{
Name: "myweb2",
Type: "apply-component",
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb2"}`)},
},
},
},
},
}
Expect(k8sClient.Create(context.Background(), app)).Should(BeNil())
appKey := types.NamespacedName{Namespace: ns.Name, Name: app.Name}
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
By("verify the first ten reconciles")
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes; i++ {
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
}
expDeployment := &v1.Deployment{}
web1Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb1"}
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(util.NotFoundMatcher{})
web2Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb2"}
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(BeNil())
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
expDeployment.Status.Replicas = 1
expDeployment.Status.ReadyReplicas = 1
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
checkApp := &v1beta1.Application{}
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationWorkflowTerminated))
})
It("application with if always in workflow sub steps", func() {
ns := corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "app-with-if-always-workflow-sub-steps",
},
}
Expect(k8sClient.Create(ctx, &ns)).Should(BeNil())
healthComponentDef := &v1beta1.ComponentDefinition{}
hCDefJson, _ := yaml.YAMLToJSON([]byte(cdDefWithHealthStatusYaml))
Expect(json.Unmarshal(hCDefJson, healthComponentDef)).Should(BeNil())
healthComponentDef.Name = "worker-with-health"
healthComponentDef.Namespace = "app-with-if-always-workflow-sub-steps"
Expect(k8sClient.Create(ctx, healthComponentDef)).Should(BeNil())
app := &v1beta1.Application{
TypeMeta: metav1.TypeMeta{
Kind: "Application",
APIVersion: "core.oam.dev/v1beta1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "app-with-if-always-workflow-sub-steps",
Namespace: "app-with-if-always-workflow-sub-steps",
},
Spec: v1beta1.ApplicationSpec{
Components: []common.ApplicationComponent{
{
Name: "myweb1",
Type: "worker-with-health",
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
},
{
Name: "myweb2",
Type: "worker-with-health",
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox","lives": "i am lives","enemies": "empty"}`)},
},
{
Name: "myweb3",
Type: "worker",
Properties: &runtime.RawExtension{Raw: []byte(`{"cmd":["sleep","1000"],"image":"busybox"}`)},
},
{
Name: "failed-step",
Type: "k8s-objects",
Properties: &runtime.RawExtension{Raw: []byte(`{"objects":[{"apiVersion":"v1","kind":"invalid","metadata":{"name":"test1"}}]}`)},
},
},
Workflow: &v1beta1.Workflow{
Steps: []v1beta1.WorkflowStep{
{
Name: "myweb1",
Type: "step-group",
SubSteps: []common.WorkflowSubStep{
{
Name: "myweb1-sub1",
Type: "apply-component",
If: "always",
DependsOn: []string{"myweb1-sub2"},
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb1"}`)},
},
{
Name: "myweb1-sub2",
Type: "apply-component",
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"failed-step"}`)},
},
{
Name: "myweb1-sub3",
Type: "apply-component",
DependsOn: []string{"myweb1-sub1"},
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb2"}`)},
},
},
},
{
Name: "myweb2",
Type: "apply-component",
If: "always",
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb2"}`)},
},
{
Name: "myweb3",
Type: "apply-component",
Properties: &runtime.RawExtension{Raw: []byte(`{"component":"myweb3"}`)},
},
},
},
},
}
Expect(k8sClient.Create(context.Background(), app)).Should(BeNil())
appKey := types.NamespacedName{Namespace: ns.Name, Name: app.Name}
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
By("verify the first ten reconciles")
for i := 0; i < custom.MaxWorkflowStepErrorRetryTimes; i++ {
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
}
expDeployment := &v1.Deployment{}
web1Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb1"}
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(util.NotFoundMatcher{})
web2Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb2"}
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
web3Key := types.NamespacedName{Namespace: ns.Name, Name: "myweb3"}
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(util.NotFoundMatcher{})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, web1Key, expDeployment)).Should(BeNil())
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(util.NotFoundMatcher{})
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(util.NotFoundMatcher{})
expDeployment.Status.Replicas = 1
expDeployment.Status.ReadyReplicas = 1
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
Expect(k8sClient.Get(ctx, web2Key, expDeployment)).Should(BeNil())
Expect(k8sClient.Get(ctx, web3Key, expDeployment)).Should(util.NotFoundMatcher{})
expDeployment.Status.Replicas = 1
expDeployment.Status.ReadyReplicas = 1
Expect(k8sClient.Status().Update(ctx, expDeployment)).Should(BeNil())
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
testutil.ReconcileOnce(reconciler, reconcile.Request{NamespacedName: appKey})
checkApp := &v1beta1.Application{}
Expect(k8sClient.Get(ctx, appKey, checkApp)).Should(BeNil())
Expect(checkApp.Status.Phase).Should(BeEquivalentTo(common.ApplicationWorkflowTerminated))
})
It("application with input/output run as dag workflow", func() {
ns := corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
@@ -122,6 +122,7 @@ func generateStep(ctx context.Context,
DependsOn: subStep.DependsOn,
Inputs: subStep.Inputs,
Outputs: subStep.Outputs,
If: subStep.If,
}
subTask, err := generateStep(ctx, app, workflowStep, taskDiscover, step.Name)
if err != nil {
+3
View File
@@ -33,6 +33,8 @@ const (
DeprecatedObjectLabelSelector featuregate.Feature = "DeprecatedObjectLabelSelector"
// LegacyResourceTrackerGC enable the gc of legacy resource tracker in managed clusters
LegacyResourceTrackerGC featuregate.Feature = "LegacyResourceTrackerGC"
// EnableSuspendOnFailure enable suspend on workflow failure
EnableSuspendOnFailure featuregate.Feature = "EnableSuspendOnFailure"
// Edge Features
@@ -45,6 +47,7 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
LegacyObjectTypeIdentifier: {Default: false, PreRelease: featuregate.Alpha},
DeprecatedObjectLabelSelector: {Default: false, PreRelease: featuregate.Alpha},
LegacyResourceTrackerGC: {Default: true, PreRelease: featuregate.Alpha},
EnableSuspendOnFailure: {Default: false, PreRelease: featuregate.Alpha},
AuthenticateApplication: {Default: false, PreRelease: featuregate.Alpha},
}
-17
View File
@@ -28,18 +28,8 @@ import (
wfContext "github.com/oam-dev/kubevela/pkg/workflow/context"
)
const (
// ReadyComponent is the key for depends on in workflow context
ReadyComponent = "readyComponent__"
)
// Input set data to parameter.
func Input(ctx wfContext.Context, paramValue *value.Value, step v1beta1.WorkflowStep) error {
for _, depend := range step.DependsOn {
if _, err := ctx.GetVar(ReadyComponent, depend); err != nil {
return errors.WithMessagef(err, "the depends on component [%s] is not ready", depend)
}
}
for _, input := range step.Inputs {
inputValue, err := ctx.GetVar(strings.Split(input.From, ".")...)
if err != nil {
@@ -66,13 +56,6 @@ func Output(ctx wfContext.Context, taskValue *value.Value, step v1beta1.Workflow
if err := json.Unmarshal(js, &o); err != nil {
return err
}
ready, err := value.NewValue(`true`, nil, "")
if err != nil {
return err
}
if err := ctx.SetVar(ready, ReadyComponent, o.Name); err != nil {
return err
}
}
for _, output := range step.Outputs {
-10
View File
@@ -43,10 +43,6 @@ func TestInput(t *testing.T) {
r.NoError(err)
err = wfCtx.SetVar(score, "foo")
r.NoError(err)
ready, err := value.NewValue(`true`, nil, "")
r.NoError(err)
err = wfCtx.SetVar(ready, ReadyComponent, "mystep")
r.NoError(err)
err = Input(wfCtx, paramValue, v1beta1.WorkflowStep{
DependsOn: []string{"mystep"},
Inputs: common.StepInputs{{
@@ -85,12 +81,6 @@ output: score: 99
s, err := result.String()
r.NoError(err)
r.Equal(s, `99
`)
ready, err := wfCtx.GetVar(ReadyComponent, "mystep")
r.NoError(err)
s, err = ready.String()
r.NoError(err)
r.Equal(s, `true
`)
}
+92 -13
View File
@@ -24,6 +24,7 @@ import (
"cuelang.org/go/cue"
"github.com/pkg/errors"
"k8s.io/apiserver/pkg/util/feature"
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
@@ -32,6 +33,7 @@ import (
"github.com/oam-dev/kubevela/pkg/cue/model/value"
"github.com/oam-dev/kubevela/pkg/cue/packages"
"github.com/oam-dev/kubevela/pkg/cue/process"
"github.com/oam-dev/kubevela/pkg/features"
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
wfContext "github.com/oam-dev/kubevela/pkg/workflow/context"
"github.com/oam-dev/kubevela/pkg/workflow/hooks"
@@ -47,6 +49,8 @@ var (
const (
// StatusReasonWait is the reason of the workflow progress condition which is Wait.
StatusReasonWait = "Wait"
// StatusReasonSkip is the reason of the workflow progress condition which is Skip.
StatusReasonSkip = "Skip"
// StatusReasonRendering is the reason of the workflow progress condition which is Rendering.
StatusReasonRendering = "Rendering"
// StatusReasonExecute is the reason of the workflow progress condition which is Execute.
@@ -59,6 +63,8 @@ const (
StatusReasonParameter = "ProcessParameter"
// StatusReasonOutput is the reason of the workflow progress condition which is Output.
StatusReasonOutput = "Output"
// StatusReasonFailedAfterRetries is the reason of the workflow progress condition which is FailedAfterRetries.
StatusReasonFailedAfterRetries = "FailedAfterRetries"
)
// LoadTaskTemplate gets the workflowStep definition from cluster and resolve it.
@@ -85,7 +91,8 @@ func (t *TaskLoader) GetTaskGenerator(ctx context.Context, name string) (wfTypes
type taskRunner struct {
name string
run func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error)
checkPending func(ctx wfContext.Context) bool
checkPending func(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool
skip func(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool)
}
// Name return step name.
@@ -99,10 +106,15 @@ func (tr *taskRunner) Run(ctx wfContext.Context, options *wfTypes.TaskRunOptions
}
// Pending check task should be executed or not.
func (tr *taskRunner) Pending(ctx wfContext.Context) bool {
return tr.checkPending(ctx)
func (tr *taskRunner) Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
return tr.checkPending(ctx, stepStatus)
}
func (tr *taskRunner) Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
return tr.skip(dependsOnPhase, stepStatus)
}
// nolint:gocyclo
func (t *TaskLoader) makeTaskGenerator(templ string) (wfTypes.TaskGenerator, error) {
return func(wfStep v1beta1.WorkflowStep, genOpt *wfTypes.GeneratorOptions) (wfTypes.TaskRunner, error) {
@@ -141,18 +153,21 @@ func (t *TaskLoader) makeTaskGenerator(templ string) (wfTypes.TaskGenerator, err
tRunner := new(taskRunner)
tRunner.name = wfStep.Name
tRunner.checkPending = func(ctx wfContext.Context) bool {
for _, depend := range wfStep.DependsOn {
if _, err := ctx.GetVar(hooks.ReadyComponent, depend); err != nil {
return true
}
tRunner.checkPending = func(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
return CheckPending(ctx, wfStep, stepStatus)
}
tRunner.skip = func(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
return exec.status(), false
}
for _, input := range wfStep.Inputs {
if _, err := ctx.GetVar(strings.Split(input.From, ".")...); err != nil {
return true
}
skip := SkipTaskRunner(&SkipOptions{
If: wfStep.If,
DependsOnPhase: dependsOnPhase,
})
if skip {
exec.Skip("")
}
return false
return exec.status(), skip
}
tRunner.run = func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error) {
if options.GetTracer == nil {
@@ -261,6 +276,7 @@ type executor struct {
terminated bool
failedAfterRetries bool
wait bool
skip bool
tracer monitorContext.Context
}
@@ -289,6 +305,13 @@ func (exec *executor) Wait(message string) {
exec.wfStatus.Message = message
}
func (exec *executor) Skip(message string) {
exec.skip = true
exec.wfStatus.Phase = common.WorkflowStepPhaseSkipped
exec.wfStatus.Reason = StatusReasonSkip
exec.wfStatus.Message = message
}
func (exec *executor) err(ctx wfContext.Context, err error, reason string) {
exec.wait = true
exec.wfStatus.Phase = common.WorkflowStepPhaseFailed
@@ -302,6 +325,7 @@ func (exec *executor) checkErrorTimes(ctx wfContext.Context) {
if times >= MaxWorkflowStepErrorRetryTimes {
exec.wait = false
exec.failedAfterRetries = true
exec.wfStatus.Reason = StatusReasonFailedAfterRetries
}
}
@@ -441,3 +465,58 @@ func NewTaskLoader(lt LoadTaskTemplate, pkgDiscover *packages.PackageDiscover, h
logLevel: logLevel,
}
}
// SkipOptions is the options of skip task runner
type SkipOptions struct {
If string
DependsOnPhase common.WorkflowStepPhase
}
// SkipTaskRunner will decide whether to skip task runner.
func SkipTaskRunner(options *SkipOptions) bool {
switch options.If {
case "always":
return false
case "":
return options.DependsOnPhase != common.WorkflowStepPhaseSucceeded
default:
// TODO:(fog) support more if cases
return false
}
}
// CheckPending checks whether to pending task run
func CheckPending(ctx wfContext.Context, step v1beta1.WorkflowStep, stepStatus map[string]common.StepStatus) bool {
for _, depend := range step.DependsOn {
if status, ok := stepStatus[depend]; ok {
if !IsStepFinish(status.Phase, status.Reason) {
return true
}
} else {
return true
}
}
for _, input := range step.Inputs {
if _, err := ctx.GetVar(strings.Split(input.From, ".")...); err != nil {
return true
}
}
return false
}
// IsStepFinish will decide whether step is finish.
func IsStepFinish(phase common.WorkflowStepPhase, reason string) bool {
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
return phase == common.WorkflowStepPhaseSucceeded
}
switch phase {
case common.WorkflowStepPhaseFailed:
return reason == StatusReasonTerminate || reason == StatusReasonFailedAfterRetries
case common.WorkflowStepPhaseSkipped:
return true
case common.WorkflowStepPhaseSucceeded:
return true
default:
return false
}
}
+44 -7
View File
@@ -36,7 +36,6 @@ import (
"github.com/oam-dev/kubevela/pkg/cue/model/value"
"github.com/oam-dev/kubevela/pkg/cue/process"
wfContext "github.com/oam-dev/kubevela/pkg/workflow/context"
"github.com/oam-dev/kubevela/pkg/workflow/hooks"
"github.com/oam-dev/kubevela/pkg/workflow/providers"
"github.com/oam-dev/kubevela/pkg/workflow/types"
)
@@ -270,6 +269,7 @@ close({
r.Equal(operation.Waiting, false)
r.Equal(operation.FailedAfterRetries, true)
r.Equal(status.Phase, common.WorkflowStepPhaseFailed)
r.Equal(status.Reason, StatusReasonFailedAfterRetries)
default:
r.Equal(operation.Waiting, true)
r.Equal(status.Phase, common.WorkflowStepPhaseFailed)
@@ -438,14 +438,14 @@ func TestPendingInputCheck(t *testing.T) {
r.NoError(err)
run, err := gen(step, &types.GeneratorOptions{})
r.NoError(err)
r.Equal(run.Pending(wfCtx), true)
r.Equal(run.Pending(wfCtx, nil), true)
score, err := value.NewValue(`
100
`, nil, "")
r.NoError(err)
err = wfCtx.SetVar(score, "score")
r.NoError(err)
r.Equal(run.Pending(wfCtx), false)
r.Equal(run.Pending(wfCtx, nil), false)
}
func TestPendingDependsOnCheck(t *testing.T) {
@@ -473,12 +473,49 @@ func TestPendingDependsOnCheck(t *testing.T) {
r.NoError(err)
run, err := gen(step, &types.GeneratorOptions{})
r.NoError(err)
r.Equal(run.Pending(wfCtx), true)
ready, err := value.NewValue("true", nil, "")
r.Equal(run.Pending(wfCtx, nil), true)
ss := map[string]common.StepStatus{
"depend": {
Phase: common.WorkflowStepPhaseSucceeded,
},
}
r.Equal(run.Pending(wfCtx, ss), false)
}
func TestSkip(t *testing.T) {
r := require.New(t)
discover := providers.NewProviders()
discover.Register("test", map[string]providers.Handler{
"ok": func(ctx wfContext.Context, v *value.Value, act types.Action) error {
return nil
},
})
step := v1beta1.WorkflowStep{
Name: "skip",
Type: "ok",
}
pCtx := process.NewContext(process.ContextData{
AppName: "myapp",
CompName: "mycomp",
Namespace: "default",
AppRevisionName: "myapp-v1",
})
tasksLoader := NewTaskLoader(mockLoadTemplate, nil, discover, 0, pCtx)
gen, err := tasksLoader.GetTaskGenerator(context.Background(), step.Type)
r.NoError(err)
err = wfCtx.SetVar(ready, hooks.ReadyComponent, "depend")
runner, err := gen(step, &types.GeneratorOptions{})
r.NoError(err)
r.Equal(run.Pending(wfCtx), false)
status, skip := runner.Skip(common.WorkflowStepPhaseFailed, nil)
r.Equal(skip, true)
r.Equal(status.Phase, common.WorkflowStepPhaseSkipped)
r.Equal(status.Reason, StatusReasonSkip)
runner2, err := gen(v1beta1.WorkflowStep{
If: "always",
Name: "test",
}, &types.GeneratorOptions{ID: "124"})
r.NoError(err)
_, skip = runner2.Skip(common.WorkflowStepPhaseFailed, nil)
r.Equal(skip, false)
}
func newWorkflowContextForTest(t *testing.T) wfContext.Context {
+99 -36
View File
@@ -22,6 +22,7 @@ import (
builtintime "time"
"github.com/pkg/errors"
"k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -29,6 +30,7 @@ import (
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
"github.com/oam-dev/kubevela/pkg/cue/packages"
"github.com/oam-dev/kubevela/pkg/cue/process"
"github.com/oam-dev/kubevela/pkg/features"
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
"github.com/oam-dev/kubevela/pkg/oam/discoverymapper"
"github.com/oam-dev/kubevela/pkg/velaql/providers/query"
@@ -73,7 +75,7 @@ func (td *taskDiscover) GetTaskGenerator(ctx context.Context, name string) (type
func suspend(step v1beta1.WorkflowStep, opt *types.GeneratorOptions) (types.TaskRunner, error) {
tr := &suspendTaskRunner{
id: opt.ID,
name: step.Name,
step: step,
wait: false,
}
@@ -92,6 +94,7 @@ func StepGroup(step v1beta1.WorkflowStep, opt *types.GeneratorOptions) (types.Ta
return &stepGroupTaskRunner{
id: opt.ID,
name: step.Name,
step: step,
subTaskRunners: opt.SubTaskRunners,
}, nil
}
@@ -119,40 +122,64 @@ func NewTaskDiscoverFromRevision(ctx monitorContext.Context, providerHandlers pr
}
type suspendTaskRunner struct {
id string
name string
wait bool
id string
step v1beta1.WorkflowStep
wait bool
phase common.WorkflowStepPhase
}
// Name return suspend step name.
func (tr *suspendTaskRunner) Name() string {
return tr.name
return tr.step.Name
}
// Run make workflow suspend.
func (tr *suspendTaskRunner) Run(ctx wfContext.Context, options *types.TaskRunOptions) (common.StepStatus, *types.Operation, error) {
if tr.wait {
tr.phase = common.WorkflowStepPhaseRunning
} else {
tr.phase = common.WorkflowStepPhaseSucceeded
}
stepStatus := common.StepStatus{
ID: tr.id,
Name: tr.name,
Name: tr.step.Name,
Type: types.WorkflowStepTypeSuspend,
Phase: common.WorkflowStepPhaseSucceeded,
}
if tr.wait {
stepStatus.Phase = common.WorkflowStepPhaseRunning
Phase: tr.phase,
}
return stepStatus, &types.Operation{Suspend: true}, nil
}
// Pending check task should be executed or not.
func (tr *suspendTaskRunner) Pending(ctx wfContext.Context) bool {
return false
func (tr *suspendTaskRunner) Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
return custom.CheckPending(ctx, tr.step, stepStatus)
}
func (tr *suspendTaskRunner) Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
status := common.StepStatus{
ID: tr.id,
Name: tr.step.Name,
Type: types.WorkflowStepTypeSuspend,
Phase: tr.phase,
}
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
return status, false
}
skip := custom.SkipTaskRunner(&custom.SkipOptions{
If: tr.step.If,
DependsOnPhase: dependsOnPhase,
})
if skip {
status.Phase = common.WorkflowStepPhaseSkipped
status.Reason = custom.StatusReasonSkip
}
return status, skip
}
type stepGroupTaskRunner struct {
id string
name string
step v1beta1.WorkflowStep
subTaskRunners []types.TaskRunner
}
@@ -161,12 +188,43 @@ func (tr *stepGroupTaskRunner) Name() string {
return tr.name
}
// Pending check task should be executed or not.
func (tr *stepGroupTaskRunner) Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
return custom.CheckPending(ctx, tr.step, stepStatus)
}
func (tr *stepGroupTaskRunner) Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
status := common.StepStatus{
ID: tr.id,
Name: tr.step.Name,
Type: types.WorkflowStepTypeStepGroup,
}
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
return status, false
}
skip := custom.SkipTaskRunner(&custom.SkipOptions{
If: tr.step.If,
DependsOnPhase: dependsOnPhase,
})
if skip {
status.Phase = common.WorkflowStepPhaseSkipped
status.Reason = custom.StatusReasonSkip
stepStatus[tr.step.Name] = common.StepStatus{
ID: tr.id,
Phase: status.Phase,
}
// return false here to set all the sub steps to skipped
return status, false
}
return status, skip
}
// Run make workflow step group.
func (tr *stepGroupTaskRunner) Run(ctx wfContext.Context, options *types.TaskRunOptions) (common.StepStatus, *types.Operation, error) {
e := options.Engine
if len(tr.subTaskRunners) > 0 {
// set sub steps to dag mode for now
e.SetParentRunner(tr.name)
// set sub steps to dag mode for now
if err := e.Run(tr.subTaskRunners, true); err != nil {
return common.StepStatus{
ID: tr.id,
@@ -178,34 +236,39 @@ func (tr *stepGroupTaskRunner) Run(ctx wfContext.Context, options *types.TaskRun
e.SetParentRunner("")
}
stepStatus := e.GetStepStatus(tr.name)
var phase common.WorkflowStepPhase
subStepPhases := make(map[common.WorkflowStepPhase]int)
status := common.StepStatus{
ID: tr.id,
Name: tr.name,
Type: types.WorkflowStepTypeStepGroup,
}
subStepCounts := make(map[string]int)
for _, subStepsStatus := range stepStatus.SubStepsStatus {
subStepPhases[subStepsStatus.Phase]++
subStepCounts[string(subStepsStatus.Phase)]++
subStepCounts[subStepsStatus.Reason]++
}
switch {
case len(stepStatus.SubStepsStatus) < len(tr.subTaskRunners):
phase = common.WorkflowStepPhaseRunning
case subStepPhases[common.WorkflowStepPhaseRunning] > 0:
phase = common.WorkflowStepPhaseRunning
case subStepPhases[common.WorkflowStepPhaseStopped] > 0:
phase = common.WorkflowStepPhaseStopped
case subStepPhases[common.WorkflowStepPhaseFailed] > 0:
phase = common.WorkflowStepPhaseFailed
status.Phase = common.WorkflowStepPhaseRunning
case subStepCounts[string(common.WorkflowStepPhaseRunning)] > 0:
status.Phase = common.WorkflowStepPhaseRunning
case subStepCounts[string(common.WorkflowStepPhaseStopped)] > 0:
status.Phase = common.WorkflowStepPhaseStopped
case subStepCounts[string(common.WorkflowStepPhaseFailed)] > 0:
status.Phase = common.WorkflowStepPhaseFailed
switch {
case subStepCounts[custom.StatusReasonFailedAfterRetries] > 0:
status.Reason = custom.StatusReasonFailedAfterRetries
case subStepCounts[custom.StatusReasonTerminate] > 0:
status.Reason = custom.StatusReasonTerminate
}
case subStepCounts[string(common.WorkflowStepPhaseSkipped)] > 0:
status.Phase = common.WorkflowStepPhaseSkipped
status.Reason = custom.StatusReasonSkip
default:
phase = common.WorkflowStepPhaseSucceeded
status.Phase = common.WorkflowStepPhaseSucceeded
}
return common.StepStatus{
ID: tr.id,
Name: tr.name,
Type: types.WorkflowStepTypeStepGroup,
Phase: phase,
}, e.GetOperation(), nil
}
// Pending check task should be executed or not.
func (tr *stepGroupTaskRunner) Pending(ctx wfContext.Context) bool {
return false
return status, e.GetOperation(), nil
}
// NewViewTaskDiscover will create a client for load task generator.
+58 -4
View File
@@ -25,6 +25,7 @@ import (
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
"github.com/oam-dev/kubevela/pkg/cue/process"
"github.com/oam-dev/kubevela/pkg/workflow/tasks/custom"
"github.com/oam-dev/kubevela/pkg/workflow/types"
@@ -82,10 +83,36 @@ func TestSuspendStep(t *testing.T) {
}
gen, err := discover.GetTaskGenerator(context.Background(), "suspend")
r.NoError(err)
runner, err := gen(v1beta1.WorkflowStep{Name: "test"}, &types.GeneratorOptions{ID: "124"})
runner, err := gen(v1beta1.WorkflowStep{
Name: "test",
DependsOn: []string{"depend"},
}, &types.GeneratorOptions{ID: "124"})
r.NoError(err)
r.Equal(runner.Name(), "test")
r.Equal(runner.Pending(nil), false)
// test pending
r.Equal(runner.Pending(nil, nil), true)
ss := map[string]common.StepStatus{
"depend": {
Phase: common.WorkflowStepPhaseSucceeded,
},
}
r.Equal(runner.Pending(nil, ss), false)
// test skip
status, skip := runner.Skip(common.WorkflowStepPhaseFailed, nil)
r.Equal(skip, true)
r.Equal(status.Phase, common.WorkflowStepPhaseSkipped)
r.Equal(status.Reason, custom.StatusReasonSkip)
runner2, err := gen(v1beta1.WorkflowStep{
If: "always",
Name: "test",
}, &types.GeneratorOptions{ID: "124"})
r.NoError(err)
_, skip = runner2.Skip(common.WorkflowStepPhaseFailed, nil)
r.Equal(skip, false)
// test run
status, act, err := runner.Run(nil, nil)
r.NoError(err)
r.Equal(act.Suspend, true)
@@ -127,11 +154,38 @@ func TestStepGroupStep(t *testing.T) {
r.NoError(err)
gen, err := discover.GetTaskGenerator(context.Background(), "stepGroup")
r.NoError(err)
runner, err := gen(v1beta1.WorkflowStep{Name: "test"}, &types.GeneratorOptions{ID: "124", SubTaskRunners: []types.TaskRunner{subRunner}})
runner, err := gen(v1beta1.WorkflowStep{
Name: "test",
DependsOn: []string{"depend"},
}, &types.GeneratorOptions{ID: "124", SubTaskRunners: []types.TaskRunner{subRunner}})
r.NoError(err)
r.Equal(runner.Name(), "test")
r.Equal(runner.Pending(nil), false)
// test pending
r.Equal(runner.Pending(nil, nil), true)
ss := map[string]common.StepStatus{
"depend": {
Phase: common.WorkflowStepPhaseSucceeded,
},
}
r.Equal(runner.Pending(nil, ss), false)
// test skip
stepStatus := make(map[string]common.StepStatus)
status, skip := runner.Skip(common.WorkflowStepPhaseFailed, stepStatus)
r.Equal(skip, false)
r.Equal(stepStatus["test"].Phase, common.WorkflowStepPhaseSkipped)
r.Equal(status.Phase, common.WorkflowStepPhaseSkipped)
r.Equal(status.Reason, custom.StatusReasonSkip)
runner2, err := gen(v1beta1.WorkflowStep{
If: "always",
Name: "test",
}, &types.GeneratorOptions{ID: "124"})
r.NoError(err)
_, skip = runner2.Skip(common.WorkflowStepPhaseFailed, stepStatus)
r.Equal(skip, false)
// test run
testCases := []struct {
name string
engine *testEngine
+3 -1
View File
@@ -30,8 +30,9 @@ import (
// TaskRunner is a task runner.
type TaskRunner interface {
Name() string
Pending(ctx wfContext.Context) bool
Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool
Run(ctx wfContext.Context, options *TaskRunOptions) (common.StepStatus, *Operation, error)
Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool)
}
// TaskDiscover is the interface to obtain the TaskGenerator。
@@ -71,6 +72,7 @@ type Operation struct {
Suspend bool
Terminated bool
Waiting bool
Skip bool
FailedAfterRetries bool
}
+236 -118
View File
@@ -25,6 +25,7 @@ import (
"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/util/feature"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
@@ -32,6 +33,7 @@ import (
oamcore "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
"github.com/oam-dev/kubevela/pkg/controller/utils"
"github.com/oam-dev/kubevela/pkg/cue/model/value"
"github.com/oam-dev/kubevela/pkg/features"
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
"github.com/oam-dev/kubevela/pkg/monitor/metrics"
"github.com/oam-dev/kubevela/pkg/oam"
@@ -41,6 +43,7 @@ import (
"github.com/oam-dev/kubevela/pkg/workflow/debug"
"github.com/oam-dev/kubevela/pkg/workflow/recorder"
wfTasks "github.com/oam-dev/kubevela/pkg/workflow/tasks"
"github.com/oam-dev/kubevela/pkg/workflow/tasks/custom"
wfTypes "github.com/oam-dev/kubevela/pkg/workflow/types"
)
@@ -61,8 +64,10 @@ const (
// backoffTimeCoefficient is the coefficient of time to wait before reconcile workflow again
backoffTimeCoefficient = 0.05
// MessageFailedAfterRetries is the message of failed after retries
MessageFailedAfterRetries = "The workflow suspends automatically because the failed times of steps have reached the limit"
// MessageTerminatedFailedAfterRetries is the message of failed after retries
MessageTerminatedFailedAfterRetries = "The workflow terminates automatically because the failed times of steps have reached the limit"
// MessageSuspendFailedAfterRetries is the message of failed after retries
MessageSuspendFailedAfterRetries = "The workflow suspends automatically because the failed times of steps have reached the limit"
// MessageInitializingWorkflow is the message of initializing workflow
MessageInitializingWorkflow = "Initializing workflow"
)
@@ -103,56 +108,24 @@ func (w *workflow) ExecuteSteps(ctx monitorContext.Context, appRev *oamcore.Appl
}
if w.app.Status.Workflow == nil || w.app.Status.Workflow.AppRevision != revAndSpecHash {
ctx.Info("Restart Workflow")
status := w.app.Status.Workflow
if status != nil && !status.Finished {
status.Terminated = true
return common.WorkflowStateTerminated, nil
}
w.app.Status.Workflow = &common.WorkflowStatus{
AppRevision: revAndSpecHash,
Mode: common.WorkflowModeStep,
StartTime: metav1.Now(),
}
w.app.Status.Workflow.Message = MessageInitializingWorkflow
if w.dagMode {
w.app.Status.Workflow.Mode = common.WorkflowModeDAG
}
// clean recorded resources info.
w.app.Status.Services = nil
w.app.Status.AppliedResources = nil
// clean conditions after render
var reservedConditions []condition.Condition
for i, cond := range w.app.Status.Conditions {
condTpy, err := common.ParseApplicationConditionType(string(cond.Type))
if err == nil {
if condTpy <= common.RenderCondition {
reservedConditions = append(reservedConditions, w.app.Status.Conditions[i])
}
}
}
w.app.Status.Conditions = reservedConditions
StepStatusCache.Delete(fmt.Sprintf("%s-%s", w.app.Name, w.app.Namespace))
wfContext.CleanupMemoryStore(w.app.Name, w.app.Namespace)
return common.WorkflowStateInitializing, nil
return w.restartWorkflow(ctx, revAndSpecHash)
}
wfStatus := w.app.Status.Workflow
cacheKey := fmt.Sprintf("%s-%s", w.app.Name, w.app.Namespace)
allTasksDone, allTasksSucceeded := w.allDone(taskRunners)
if wfStatus.Finished {
StepStatusCache.Delete(cacheKey)
return common.WorkflowStateFinished, nil
}
if wfStatus.Terminated {
if checkWorkflowTerminated(wfStatus, allTasksDone) {
return common.WorkflowStateTerminated, nil
}
if wfStatus.Suspend {
return common.WorkflowStateSuspended, nil
}
allTasksDone := w.allDone(taskRunners)
if allTasksDone {
if allTasksSucceeded {
return common.WorkflowStateSucceeded, nil
}
@@ -171,15 +144,7 @@ func (w *workflow) ExecuteSteps(ctx monitorContext.Context, appRev *oamcore.Appl
}
}
e := &engine{
status: wfStatus,
monitorCtx: ctx,
app: w.app,
wfCtx: wfCtx,
cli: w.cli,
debug: w.debug,
rk: w.rk,
}
e := newEngine(ctx, wfCtx, w, wfStatus)
err = e.Run(taskRunners, w.dagMode)
if err != nil {
@@ -191,15 +156,19 @@ func (w *workflow) ExecuteSteps(ctx monitorContext.Context, appRev *oamcore.Appl
e.checkWorkflowStatusMessage(wfStatus)
StepStatusCache.Store(cacheKey, len(wfStatus.Steps))
allTasksDone, allTasksSucceeded = w.allDone(taskRunners)
if wfStatus.Terminated {
wfContext.CleanupMemoryStore(e.app.Name, e.app.Namespace)
return common.WorkflowStateTerminated, nil
e.cleanBackoffTimesForTerminated()
if checkWorkflowTerminated(wfStatus, allTasksDone) {
wfContext.CleanupMemoryStore(e.app.Name, e.app.Namespace)
return common.WorkflowStateTerminated, nil
}
}
if wfStatus.Suspend {
wfContext.CleanupMemoryStore(e.app.Name, e.app.Namespace)
return common.WorkflowStateSuspended, nil
}
if w.allDone(taskRunners) {
if allTasksSucceeded {
wfStatus.Message = string(common.WorkflowStateSucceeded)
return common.WorkflowStateSucceeded, nil
}
@@ -207,6 +176,88 @@ func (w *workflow) ExecuteSteps(ctx monitorContext.Context, appRev *oamcore.Appl
return common.WorkflowStateExecuting, nil
}
func checkWorkflowTerminated(wfStatus *common.WorkflowStatus, allTasksDone bool) bool {
return (wfStatus.Terminated && allTasksDone) || (wfStatus.Terminated && wfStatus.Suspend)
}
func (w *workflow) restartWorkflow(ctx monitorContext.Context, revAndSpecHash string) (common.WorkflowState, error) {
ctx.Info("Restart Workflow")
status := w.app.Status.Workflow
if status != nil && !status.Finished {
status.Terminated = true
return common.WorkflowStateTerminated, nil
}
w.app.Status.Workflow = &common.WorkflowStatus{
AppRevision: revAndSpecHash,
Mode: common.WorkflowModeStep,
StartTime: metav1.Now(),
}
w.app.Status.Workflow.Message = MessageInitializingWorkflow
if w.dagMode {
w.app.Status.Workflow.Mode = common.WorkflowModeDAG
}
// clean recorded resources info.
w.app.Status.Services = nil
w.app.Status.AppliedResources = nil
// clean conditions after render
var reservedConditions []condition.Condition
for i, cond := range w.app.Status.Conditions {
condTpy, err := common.ParseApplicationConditionType(string(cond.Type))
if err == nil {
if condTpy <= common.RenderCondition {
reservedConditions = append(reservedConditions, w.app.Status.Conditions[i])
}
}
}
w.app.Status.Conditions = reservedConditions
StepStatusCache.Delete(fmt.Sprintf("%s-%s", w.app.Name, w.app.Namespace))
wfContext.CleanupMemoryStore(w.app.Name, w.app.Namespace)
return common.WorkflowStateInitializing, nil
}
func newEngine(ctx monitorContext.Context, wfCtx wfContext.Context, w *workflow, wfStatus *common.WorkflowStatus) *engine {
stepStatus := make(map[string]common.StepStatus)
for _, ss := range wfStatus.Steps {
stepStatus[ss.Name] = common.StepStatus{
Phase: ss.Phase,
ID: ss.ID,
Reason: ss.Reason,
}
for _, sss := range ss.SubStepsStatus {
stepStatus[sss.Name] = common.StepStatus{
Phase: sss.Phase,
ID: sss.ID,
Reason: ss.Reason,
}
}
}
stepDependsOn := make(map[string][]string)
if w.app.Spec.Workflow != nil {
for _, step := range w.app.Spec.Workflow.Steps {
stepDependsOn[step.Name] = append(stepDependsOn[step.Name], step.DependsOn...)
for _, sub := range step.SubSteps {
stepDependsOn[sub.Name] = append(stepDependsOn[sub.Name], sub.DependsOn...)
}
}
} else {
for _, comp := range w.app.Spec.Components {
stepDependsOn[comp.Name] = append(stepDependsOn[comp.Name], comp.DependsOn...)
}
}
return &engine{
status: wfStatus,
monitorCtx: ctx,
app: w.app,
wfCtx: wfCtx,
cli: w.cli,
debug: w.debug,
rk: w.rk,
stepStatus: stepStatus,
stepDependsOn: stepDependsOn,
}
}
// Trace record the workflow execute history.
func (w *workflow) Trace() error {
if DisableRecorder {
@@ -294,21 +345,23 @@ func (w *workflow) getWorkflowStepByName(name string) oamcore.WorkflowStep {
return oamcore.WorkflowStep{}
}
func (w *workflow) allDone(taskRunners []wfTypes.TaskRunner) bool {
func (w *workflow) allDone(taskRunners []wfTypes.TaskRunner) (bool, bool) {
success := true
status := w.app.Status.Workflow
for _, t := range taskRunners {
done := false
for _, ss := range status.Steps {
if ss.Name == t.Name() {
done = ss.Phase == common.WorkflowStepPhaseSucceeded
done = custom.IsStepFinish(ss.Phase, ss.Reason)
success = done && (ss.Phase == common.WorkflowStepPhaseSucceeded)
break
}
}
if !done {
return false
return false, false
}
}
return true
return true, success
}
func (w *workflow) makeContext(appName string) (wfCtx wfContext.Context, err error) {
@@ -432,18 +485,15 @@ func (e *engine) runAsDAG(taskRunners []wfTypes.TaskRunner) error {
wfCtx := e.wfCtx
done := true
for _, tRunner := range taskRunners {
ready := false
finish := false
var stepID string
for _, ss := range e.status.Steps {
if ss.Name == tRunner.Name() {
stepID = ss.ID
ready = ss.Phase == common.WorkflowStepPhaseSucceeded
break
}
if status, ok := e.stepStatus[tRunner.Name()]; ok {
stepID = status.ID
finish = custom.IsStepFinish(status.Phase, status.Reason)
}
if !ready {
if !finish {
done = false
if tRunner.Pending(wfCtx) {
if tRunner.Pending(wfCtx, e.stepStatus) {
pendingTasks = append(pendingTasks, tRunner)
continue
}
@@ -479,45 +529,37 @@ func (e *engine) Run(taskRunners []wfTypes.TaskRunner, dag bool) error {
if dag {
err = e.runAsDAG(taskRunners)
} else {
err = e.steps(e.todoByIndex(taskRunners), dag)
err = e.steps(taskRunners, dag)
}
e.checkFailedAfterRetries()
e.setNextExecuteTime()
return err
}
func (e *engine) checkWorkflowStatusMessage(wfStatus *common.WorkflowStatus) {
if !e.waiting && e.failedAfterRetries {
e.status.Message = MessageFailedAfterRetries
return
}
if wfStatus.Terminated {
switch {
case !e.waiting && e.failedAfterRetries && feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure):
e.status.Message = MessageSuspendFailedAfterRetries
case e.failedAfterRetries && !feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure):
e.status.Message = MessageTerminatedFailedAfterRetries
case wfStatus.Terminated:
e.status.Message = string(common.WorkflowStateTerminated)
}
if wfStatus.Suspend {
case wfStatus.Suspend:
e.status.Message = string(common.WorkflowStateSuspended)
default:
}
}
func (e *engine) todoByIndex(taskRunners []wfTypes.TaskRunner) []wfTypes.TaskRunner {
index := 0
for _, t := range taskRunners {
for _, ss := range e.status.Steps {
if ss.Name == t.Name() {
if ss.Phase == common.WorkflowStepPhaseSucceeded {
index++
}
break
}
}
}
return taskRunners[index:]
}
func (e *engine) steps(taskRunners []wfTypes.TaskRunner, dag bool) error {
wfCtx := e.wfCtx
for _, runner := range taskRunners {
var err error
for index, runner := range taskRunners {
if status, ok := e.stepStatus[runner.Name()]; ok {
if custom.IsStepFinish(status.Phase, status.Reason) {
continue
}
}
options := &wfTypes.TaskRunOptions{
GetTracer: func(id string, stepStatus oamcore.WorkflowStep) monitorContext.Context {
return e.monitorCtx.Fork(id, monitorContext.DurationMetric(func(v float64) {
@@ -535,42 +577,41 @@ func (e *engine) steps(taskRunners []wfTypes.TaskRunner, dag bool) error {
return nil
}
}
status, operation, err := runner.Run(wfCtx, options)
if err != nil {
return err
var status common.StepStatus
operation := &wfTypes.Operation{}
skip := false
status, skip = runner.Skip(e.findDependPhase(taskRunners, index, dag), e.stepStatus)
if !skip {
status, operation, err = runner.Run(wfCtx, options)
if err != nil {
return err
}
}
e.updateStepStatus(status)
e.failedAfterRetries = e.failedAfterRetries || operation.FailedAfterRetries
e.waiting = e.waiting || operation.Waiting
if status.Phase == common.WorkflowStepPhaseSucceeded || (status.Phase == common.WorkflowStepPhaseRunning && status.Type == wfTypes.WorkflowStepTypeSuspend) {
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffReason, status.ID)
if err := wfCtx.Commit(); err != nil {
return errors.WithMessage(err, "commit workflow context")
// for the suspend step with duration, there's no need to increase the backoff time in reconcile when it's still running
if !custom.IsStepFinish(status.Phase, status.Reason) && !isWaitSuspendStep(status) {
if err := handleBackoffTimes(wfCtx, status, false); err != nil {
return err
}
e.finishStep(operation)
if e.needStop() {
return nil
if dag {
continue
}
continue
return nil
}
// clear the backoff time when the step is finished
if err := handleBackoffTimes(wfCtx, status, true); err != nil {
return err
}
if val, exists := wfCtx.GetValueInMemory(wfTypes.ContextPrefixBackoffReason, status.ID); !exists || val != status.Message {
wfCtx.SetValueInMemory(status.Message, wfTypes.ContextPrefixBackoffReason, status.ID)
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
e.finishStep(operation)
if e.needStop() {
return nil
}
wfCtx.IncreaseCountValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
if err := wfCtx.Commit(); err != nil {
return errors.WithMessage(err, "commit workflow context")
}
if dag {
continue
}
e.checkFailedAfterRetries()
return nil
}
return nil
}
@@ -586,12 +627,14 @@ type engine struct {
cli client.Client
rk resourcekeeper.ResourceKeeper
parentRunner string
stepStatus map[string]common.StepStatus
stepDependsOn map[string][]string
}
func (e *engine) finishStep(operation *wfTypes.Operation) {
if operation != nil {
e.status.Suspend = operation.Suspend
e.status.Terminated = operation.Terminated
e.status.Terminated = e.status.Terminated || operation.Terminated
}
}
@@ -647,17 +690,24 @@ func (e *engine) updateStepStatus(status common.StepStatus) {
e.status.Steps = append(e.status.Steps, common.WorkflowStepStatus{StepStatus: status})
}
}
e.stepStatus[status.Name] = status
}
func (e *engine) checkFailedAfterRetries() {
if !e.waiting && e.failedAfterRetries {
if !e.waiting && e.failedAfterRetries && feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
e.status.Suspend = true
}
if e.failedAfterRetries && !feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
e.status.Terminated = true
}
}
func (e *engine) needStop() bool {
e.checkFailedAfterRetries()
return e.status.Suspend || e.status.Terminated
if feature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
e.checkFailedAfterRetries()
}
// if the workflow is terminated, we still need to execute all the remaining steps
return e.status.Suspend
}
// ComputeWorkflowRevisionHash compute workflow revision.
@@ -678,7 +728,75 @@ func ComputeWorkflowRevisionHash(rev string, app *oamcore.Application) (string,
// IsFailedAfterRetry check if application is hang due to FailedAfterRetry
func IsFailedAfterRetry(app *oamcore.Application) bool {
return app.Status.Workflow != nil && app.Status.Workflow.Message == MessageFailedAfterRetries
return app.Status.Workflow != nil && (app.Status.Workflow.Message == MessageTerminatedFailedAfterRetries || app.Status.Workflow.Message == MessageSuspendFailedAfterRetries)
}
func (e *engine) findDependPhase(taskRunners []wfTypes.TaskRunner, index int, dag bool) common.WorkflowStepPhase {
if e.parentRunner != "" {
if status, ok := e.stepStatus[e.parentRunner]; ok && status.Phase == common.WorkflowStepPhaseSkipped {
return common.WorkflowStepPhaseSkipped
}
}
if dag {
return e.findDependsOnPhase(taskRunners[index].Name())
}
if index < 1 {
return common.WorkflowStepPhaseSucceeded
}
for i := index - 1; i >= 0; i-- {
if e.stepStatus[taskRunners[i].Name()].Phase != common.WorkflowStepPhaseSucceeded {
return e.stepStatus[taskRunners[i].Name()].Phase
}
}
return e.stepStatus[taskRunners[index-1].Name()].Phase
}
func (e *engine) findDependsOnPhase(name string) common.WorkflowStepPhase {
for _, dependsOn := range e.stepDependsOn[name] {
if e.stepStatus[dependsOn].Phase != common.WorkflowStepPhaseSucceeded {
return e.stepStatus[dependsOn].Phase
}
if result := e.findDependsOnPhase(dependsOn); result != common.WorkflowStepPhaseSucceeded {
return result
}
}
return common.WorkflowStepPhaseSucceeded
}
func isWaitSuspendStep(step common.StepStatus) bool {
return step.Type == wfTypes.WorkflowStepTypeSuspend && step.Phase == common.WorkflowStepPhaseRunning
}
func handleBackoffTimes(wfCtx wfContext.Context, status common.StepStatus, clear bool) error {
if clear {
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffReason, status.ID)
} else {
if val, exists := wfCtx.GetValueInMemory(wfTypes.ContextPrefixBackoffReason, status.ID); !exists || val != status.Message {
wfCtx.SetValueInMemory(status.Message, wfTypes.ContextPrefixBackoffReason, status.ID)
wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
}
wfCtx.IncreaseCountValueInMemory(wfTypes.ContextPrefixBackoffTimes, status.ID)
}
if err := wfCtx.Commit(); err != nil {
return errors.WithMessage(err, "commit workflow context")
}
return nil
}
func (e *engine) cleanBackoffTimesForTerminated() {
for _, ss := range e.status.Steps {
for _, sub := range ss.SubStepsStatus {
if sub.Reason == custom.StatusReasonTerminate {
e.wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, sub.ID)
e.wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffReason, sub.ID)
}
}
if ss.Reason == custom.StatusReasonTerminate {
e.wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffTimes, ss.ID)
e.wfCtx.DeleteValueInMemory(wfTypes.ContextPrefixBackoffReason, ss.ID)
}
}
}
func (e *engine) GetStepStatus(stepName string) common.WorkflowStepStatus {
+410 -36
View File
@@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"math"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -29,14 +30,18 @@ import (
corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"sigs.k8s.io/yaml"
"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
oamcore "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
"github.com/oam-dev/kubevela/pkg/cue/model/value"
"github.com/oam-dev/kubevela/pkg/features"
monitorContext "github.com/oam-dev/kubevela/pkg/monitor/context"
wfContext "github.com/oam-dev/kubevela/pkg/workflow/context"
"github.com/oam-dev/kubevela/pkg/workflow/tasks"
"github.com/oam-dev/kubevela/pkg/workflow/tasks/custom"
wfTypes "github.com/oam-dev/kubevela/pkg/workflow/types"
)
@@ -226,6 +231,170 @@ var _ = Describe("Test Workflow", func() {
})).Should(BeEquivalentTo(""))
})
It("Workflow test skipped with sub steps", func() {
By("Test skipped with step group")
app, runners := makeTestCase([]oamcore.WorkflowStep{
{
Name: "s1",
Type: "failed-after-retries",
},
{
Name: "s2",
Type: "step-group",
SubSteps: []common.WorkflowSubStep{
{
Name: "s2-sub1",
Type: "success",
},
{
Name: "s2-sub2",
Type: "failed",
},
},
},
{
Name: "s3",
Type: "success",
},
})
wf := NewWorkflow(app, k8sClient, common.WorkflowModeStep, false, nil)
ctx := monitorContext.NewTraceContext(context.Background(), "test-app")
state, err := wf.ExecuteSteps(ctx, revision, runners)
Expect(err).ToNot(HaveOccurred())
Expect(state).Should(BeEquivalentTo(common.WorkflowStateInitializing))
state, err = wf.ExecuteSteps(ctx, revision, runners)
Expect(err).ToNot(HaveOccurred())
Expect(state).Should(BeEquivalentTo(common.WorkflowStateTerminated))
app.Status.Workflow.ContextBackend = nil
cleanStepTimeStamp(app.Status.Workflow)
Expect(cmp.Diff(*app.Status.Workflow, common.WorkflowStatus{
AppRevision: app.Status.Workflow.AppRevision,
Mode: common.WorkflowModeStep,
Terminated: true,
Message: string(MessageTerminatedFailedAfterRetries),
Steps: []common.WorkflowStepStatus{{
StepStatus: common.StepStatus{
Name: "s1",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
}, {
StepStatus: common.StepStatus{
Name: "s2",
Type: "step-group",
Phase: common.WorkflowStepPhaseSkipped,
Reason: custom.StatusReasonSkip,
},
SubStepsStatus: []common.WorkflowSubStepStatus{{
StepStatus: common.StepStatus{
Name: "s2-sub1",
Type: "success",
Phase: common.WorkflowStepPhaseSkipped,
Reason: custom.StatusReasonSkip,
},
}, {
StepStatus: common.StepStatus{
Name: "s2-sub2",
Type: "failed",
Phase: common.WorkflowStepPhaseSkipped,
Reason: custom.StatusReasonSkip,
},
}},
}, {
StepStatus: common.StepStatus{
Name: "s3",
Type: "success",
Phase: common.WorkflowStepPhaseSkipped,
Reason: custom.StatusReasonSkip,
},
}},
})).Should(BeEquivalentTo(""))
})
It("Workflow test if-always with sub steps", func() {
By("Test if-always with step group")
app, runners := makeTestCase([]oamcore.WorkflowStep{
{
Name: "s1",
Type: "failed-after-retries",
},
{
Name: "s2",
If: "always",
Type: "step-group",
SubSteps: []common.WorkflowSubStep{
{
Name: "s2-sub1",
DependsOn: []string{"s2-sub2"},
If: "always",
Type: "success",
},
{
Name: "s2-sub2",
Type: "failed-after-retries",
},
},
},
{
Name: "s3",
Type: "success",
},
})
wf := NewWorkflow(app, k8sClient, common.WorkflowModeStep, false, nil)
ctx := monitorContext.NewTraceContext(context.Background(), "test-app")
state, err := wf.ExecuteSteps(ctx, revision, runners)
Expect(err).ToNot(HaveOccurred())
Expect(state).Should(BeEquivalentTo(common.WorkflowStateInitializing))
state, err = wf.ExecuteSteps(ctx, revision, runners)
Expect(err).ToNot(HaveOccurred())
Expect(state).Should(BeEquivalentTo(common.WorkflowStateTerminated))
app.Status.Workflow.ContextBackend = nil
cleanStepTimeStamp(app.Status.Workflow)
Expect(cmp.Diff(*app.Status.Workflow, common.WorkflowStatus{
AppRevision: app.Status.Workflow.AppRevision,
Mode: common.WorkflowModeStep,
Terminated: true,
Message: string(MessageTerminatedFailedAfterRetries),
Steps: []common.WorkflowStepStatus{{
StepStatus: common.StepStatus{
Name: "s1",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
}, {
StepStatus: common.StepStatus{
Name: "s2",
Type: "step-group",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
SubStepsStatus: []common.WorkflowSubStepStatus{{
StepStatus: common.StepStatus{
Name: "s2-sub1",
Type: "success",
Phase: common.WorkflowStepPhaseSucceeded,
},
}, {
StepStatus: common.StepStatus{
Name: "s2-sub2",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
}},
}, {
StepStatus: common.StepStatus{
Name: "s3",
Type: "success",
Phase: common.WorkflowStepPhaseSkipped,
Reason: custom.StatusReasonSkip,
},
}},
})).Should(BeEquivalentTo(""))
})
It("Workflow test success with sub steps", func() {
By("Test success with step group")
app, runners := makeTestCase([]oamcore.WorkflowStep{
@@ -352,8 +521,9 @@ var _ = Describe("Test Workflow", func() {
})).Should(BeEquivalentTo(""))
})
It("Workflow test for failed after retries", func() {
By("Test failed-after-retries in StepByStep mode")
It("Workflow test for failed after retries with suspend", func() {
By("Test failed-after-retries in StepByStep mode with suspend")
defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
app, runners := makeTestCase([]oamcore.WorkflowStep{
{
Name: "s1",
@@ -384,7 +554,7 @@ var _ = Describe("Test Workflow", func() {
Expect(cmp.Diff(*workflowStatus, common.WorkflowStatus{
AppRevision: workflowStatus.AppRevision,
Mode: common.WorkflowModeStep,
Message: MessageFailedAfterRetries,
Message: MessageSuspendFailedAfterRetries,
Suspend: true,
Steps: []common.WorkflowStepStatus{{
StepStatus: common.StepStatus{
@@ -394,14 +564,15 @@ var _ = Describe("Test Workflow", func() {
},
}, {
StepStatus: common.StepStatus{
Name: "s2",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Name: "s2",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
}},
})).Should(BeEquivalentTo(""))
By("Test failed-after-retries in DAG mode")
By("Test failed-after-retries in DAG mode with suspend")
app, runners = makeTestCase([]oamcore.WorkflowStep{
{
Name: "s1",
@@ -432,7 +603,7 @@ var _ = Describe("Test Workflow", func() {
Expect(cmp.Diff(*workflowStatus, common.WorkflowStatus{
AppRevision: workflowStatus.AppRevision,
Mode: common.WorkflowModeDAG,
Message: MessageFailedAfterRetries,
Message: MessageSuspendFailedAfterRetries,
Suspend: true,
Steps: []common.WorkflowStepStatus{{
StepStatus: common.StepStatus{
@@ -442,9 +613,10 @@ var _ = Describe("Test Workflow", func() {
},
}, {
StepStatus: common.StepStatus{
Name: "s2",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Name: "s2",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
}, {
StepStatus: common.StepStatus{
@@ -456,8 +628,183 @@ var _ = Describe("Test Workflow", func() {
})).Should(BeEquivalentTo(""))
})
It("Workflow test if always", func() {
By("Test if always in StepByStep mode")
app, runners := makeTestCase([]oamcore.WorkflowStep{
{
Name: "s1",
Type: "success",
},
{
Name: "s2",
Type: "failed-after-retries",
},
{
Name: "s3",
If: "always",
Type: "success",
},
{
Name: "s4",
Type: "success",
},
{
Name: "s5",
If: "always",
Type: "success",
},
})
ctx := monitorContext.NewTraceContext(context.Background(), "test-app")
wf := NewWorkflow(app, k8sClient, common.WorkflowModeStep, false, nil)
state, err := wf.ExecuteSteps(ctx, revision, runners)
Expect(err).ToNot(HaveOccurred())
Expect(state).Should(BeEquivalentTo(common.WorkflowStateInitializing))
state, err = wf.ExecuteSteps(ctx, revision, runners)
Expect(err).ToNot(HaveOccurred())
Expect(state).Should(BeEquivalentTo(common.WorkflowStateTerminated))
workflowStatus := app.Status.Workflow
Expect(workflowStatus.ContextBackend.Name).Should(BeEquivalentTo("workflow-" + app.Name + "-context"))
workflowStatus.ContextBackend = nil
cleanStepTimeStamp(workflowStatus)
Expect(cmp.Diff(*workflowStatus, common.WorkflowStatus{
AppRevision: workflowStatus.AppRevision,
Mode: common.WorkflowModeStep,
Message: MessageTerminatedFailedAfterRetries,
Suspend: false,
Terminated: true,
Steps: []common.WorkflowStepStatus{{
StepStatus: common.StepStatus{
Name: "s1",
Type: "success",
Phase: common.WorkflowStepPhaseSucceeded,
},
}, {
StepStatus: common.StepStatus{
Name: "s2",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
}, {
StepStatus: common.StepStatus{
Name: "s3",
Type: "success",
Phase: common.WorkflowStepPhaseSucceeded,
},
}, {
StepStatus: common.StepStatus{
Name: "s4",
Type: "success",
Phase: common.WorkflowStepPhaseSkipped,
Reason: custom.StatusReasonSkip,
},
}, {
StepStatus: common.StepStatus{
Name: "s5",
Type: "success",
Phase: common.WorkflowStepPhaseSucceeded,
},
}},
})).Should(BeEquivalentTo(""))
By("Test if always in DAG mode")
app, runners = makeTestCase([]oamcore.WorkflowStep{
{
Name: "s1",
Type: "success",
},
{
Name: "s2",
Type: "failed-after-retries",
},
{
Name: "s3",
DependsOn: []string{"s2"},
If: "always",
Type: "success",
},
{
Name: "s4",
DependsOn: []string{"s3"},
Type: "success",
},
{
Name: "s5",
DependsOn: []string{"s3"},
If: "always",
Type: "success",
},
{
Name: "s6",
DependsOn: []string{"s1", "s5"},
Type: "success",
},
})
ctx = monitorContext.NewTraceContext(context.Background(), "test-app")
wf = NewWorkflow(app, k8sClient, common.WorkflowModeDAG, false, nil)
state, err = wf.ExecuteSteps(ctx, revision, runners)
Expect(err).ToNot(HaveOccurred())
Expect(state).Should(BeEquivalentTo(common.WorkflowStateInitializing))
state, err = wf.ExecuteSteps(ctx, revision, runners)
Expect(err).ToNot(HaveOccurred())
Expect(state).Should(BeEquivalentTo(common.WorkflowStateTerminated))
workflowStatus = app.Status.Workflow
Expect(workflowStatus.ContextBackend.Name).Should(BeEquivalentTo("workflow-" + app.Name + "-context"))
workflowStatus.ContextBackend = nil
cleanStepTimeStamp(workflowStatus)
Expect(cmp.Diff(*workflowStatus, common.WorkflowStatus{
AppRevision: workflowStatus.AppRevision,
Mode: common.WorkflowModeDAG,
Message: MessageTerminatedFailedAfterRetries,
Terminated: true,
Steps: []common.WorkflowStepStatus{{
StepStatus: common.StepStatus{
Name: "s1",
Type: "success",
Phase: common.WorkflowStepPhaseSucceeded,
},
}, {
StepStatus: common.StepStatus{
Name: "s2",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
}, {
StepStatus: common.StepStatus{
Name: "s3",
Type: "success",
Phase: common.WorkflowStepPhaseSucceeded,
},
}, {
StepStatus: common.StepStatus{
Name: "s4",
Type: "success",
Phase: common.WorkflowStepPhaseSkipped,
Reason: custom.StatusReasonSkip,
},
}, {
StepStatus: common.StepStatus{
Name: "s5",
Type: "success",
Phase: common.WorkflowStepPhaseSucceeded,
},
}, {
StepStatus: common.StepStatus{
Name: "s6",
Type: "success",
Phase: common.WorkflowStepPhaseSkipped,
Reason: custom.StatusReasonSkip,
},
}},
})).Should(BeEquivalentTo(""))
})
It("Test failed after retries with sub steps", func() {
By("Test failed-after-retries with step group in StepByStep mode")
defer featuregatetesting.SetFeatureGateDuringTest(&testing.T{}, utilfeature.DefaultFeatureGate, features.EnableSuspendOnFailure, true)()
app, runners := makeTestCase([]oamcore.WorkflowStep{
{
Name: "s1",
@@ -495,7 +842,7 @@ var _ = Describe("Test Workflow", func() {
Expect(cmp.Diff(*app.Status.Workflow, common.WorkflowStatus{
AppRevision: app.Status.Workflow.AppRevision,
Mode: common.WorkflowModeStep,
Message: MessageFailedAfterRetries,
Message: MessageSuspendFailedAfterRetries,
Suspend: true,
Steps: []common.WorkflowStepStatus{{
StepStatus: common.StepStatus{
@@ -505,9 +852,10 @@ var _ = Describe("Test Workflow", func() {
},
}, {
StepStatus: common.StepStatus{
Name: "s2",
Type: "step-group",
Phase: common.WorkflowStepPhaseFailed,
Name: "s2",
Type: "step-group",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
SubStepsStatus: []common.WorkflowSubStepStatus{{
StepStatus: common.StepStatus{
@@ -517,9 +865,10 @@ var _ = Describe("Test Workflow", func() {
},
}, {
StepStatus: common.StepStatus{
Name: "s2-sub2",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Name: "s2-sub2",
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
},
}},
}},
@@ -1031,11 +1380,11 @@ func makeTestCase(steps []oamcore.WorkflowStep) (*oamcore.Application, []wfTypes
if step.SubSteps != nil {
subStepRunners := []wfTypes.TaskRunner{}
for _, subStep := range step.SubSteps {
subStepRunners = append(subStepRunners, makeRunner(subStep.Name, subStep.Type, nil))
subStepRunners = append(subStepRunners, makeRunner(subStep.Name, subStep.Type, subStep.If, subStep.DependsOn, nil))
}
runners = append(runners, makeRunner(step.Name, step.Type, subStepRunners))
runners = append(runners, makeRunner(step.Name, step.Type, step.If, step.DependsOn, subStepRunners))
} else {
runners = append(runners, makeRunner(step.Name, step.Type, nil))
runners = append(runners, makeRunner(step.Name, step.Type, step.If, step.DependsOn, nil))
}
}
return app, runners
@@ -1043,8 +1392,26 @@ func makeTestCase(steps []oamcore.WorkflowStep) (*oamcore.Application, []wfTypes
var pending bool
func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wfTypes.TaskRunner {
func makeRunner(name, tpy, ifDecl string, dependsOn []string, subTaskRunners []wfTypes.TaskRunner) wfTypes.TaskRunner {
var run func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error)
skip := func(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
status := common.StepStatus{
Name: name,
Type: tpy,
}
if utilfeature.DefaultMutableFeatureGate.Enabled(features.EnableSuspendOnFailure) {
return status, false
}
skip := custom.SkipTaskRunner(&custom.SkipOptions{
If: ifDecl,
DependsOnPhase: dependsOnPhase,
})
if skip {
status.Phase = common.WorkflowStepPhaseSkipped
status.Reason = custom.StatusReasonSkip
}
return status, skip
}
switch tpy {
case "suspend":
run = func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error) {
@@ -1085,9 +1452,10 @@ func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wf
case "failed-after-retries":
run = func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error) {
return common.StepStatus{
Name: name,
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Name: name,
Type: "failed-after-retries",
Phase: common.WorkflowStepPhaseFailed,
Reason: custom.StatusReasonFailedAfterRetries,
}, &wfTypes.Operation{
FailedAfterRetries: true,
}, nil
@@ -1111,8 +1479,14 @@ func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wf
}, &wfTypes.Operation{}, err
}
case "step-group":
group, _ := tasks.StepGroup(oamcore.WorkflowStep{Name: name}, &wfTypes.GeneratorOptions{SubTaskRunners: subTaskRunners})
group, _ := tasks.StepGroup(oamcore.WorkflowStep{
Name: name,
If: ifDecl,
Type: tpy,
DependsOn: dependsOn,
}, &wfTypes.GeneratorOptions{SubTaskRunners: subTaskRunners})
run = group.Run
skip = group.Skip
default:
run = func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error) {
return common.StepStatus{
@@ -1126,7 +1500,7 @@ func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wf
return &testTaskRunner{
name: name,
run: run,
checkPending: func(ctx wfContext.Context) bool {
checkPending: func(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
if tpy != "pending" {
return false
}
@@ -1135,7 +1509,7 @@ func makeRunner(name string, tpy string, subTaskRunners []wfTypes.TaskRunner) wf
}
return false
},
subTaskRunners: subTaskRunners,
skip: skip,
}
}
@@ -1156,10 +1530,10 @@ metadata:
)
type testTaskRunner struct {
name string
run func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error)
checkPending func(ctx wfContext.Context) bool
subTaskRunners []wfTypes.TaskRunner
name string
run func(ctx wfContext.Context, options *wfTypes.TaskRunOptions) (common.StepStatus, *wfTypes.Operation, error)
checkPending func(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool
skip func(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool)
}
// Name return step name.
@@ -1173,12 +1547,12 @@ func (tr *testTaskRunner) Run(ctx wfContext.Context, options *wfTypes.TaskRunOpt
}
// Pending check task should be executed or not.
func (tr *testTaskRunner) Pending(ctx wfContext.Context) bool {
return tr.checkPending(ctx)
func (tr *testTaskRunner) Pending(ctx wfContext.Context, stepStatus map[string]common.StepStatus) bool {
return tr.checkPending(ctx, stepStatus)
}
func (tr *testTaskRunner) SubTaskRunners() []wfTypes.TaskRunner {
return tr.subTaskRunners
func (tr *testTaskRunner) Skip(dependsOnPhase common.WorkflowStepPhase, stepStatus map[string]common.StepStatus) (common.StepStatus, bool) {
return tr.skip(dependsOnPhase, stepStatus)
}
func cleanStepTimeStamp(wfStatus *common.WorkflowStatus) {
+5 -6
View File
@@ -29,6 +29,7 @@ import (
oamcommon "github.com/oam-dev/kubevela/apis/core.oam.dev/common"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
"github.com/oam-dev/kubevela/apis/types"
"github.com/oam-dev/kubevela/pkg/apiserver/domain/service"
"github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha2/application"
"github.com/oam-dev/kubevela/pkg/controller/utils"
"github.com/oam-dev/kubevela/pkg/oam"
@@ -170,7 +171,7 @@ func NewWorkflowTerminateCommand(c common.Args, ioStream cmdutil.IOStreams) *cob
if err != nil {
return err
}
err = terminateWorkflow(client, app)
err = TerminateWorkflow(client, app)
if err != nil {
return err
}
@@ -286,11 +287,9 @@ func resumeWorkflow(kubecli client.Client, app *v1beta1.Application) error {
return nil
}
func terminateWorkflow(kubecli client.Client, app *v1beta1.Application) error {
// set the workflow terminated to true
app.Status.Workflow.Terminated = true
if err := kubecli.Status().Patch(context.TODO(), app, client.Merge); err != nil {
// TerminateWorkflow terminate workflow
func TerminateWorkflow(kubecli client.Client, app *v1beta1.Application) error {
if err := service.TerminateWorkflow(context.TODO(), kubecli, app); err != nil {
return err
}
+6 -6
View File
@@ -122,15 +122,15 @@ var _ = Describe("Application Normal tests", func() {
}, time.Second*5, time.Millisecond*500).Should(Succeed())
}
verifyApplicationWorkflowSuspending := func(ns, appName string) {
verifyApplicationWorkflowTerminated := func(ns, appName string) {
var testApp v1beta1.Application
Eventually(func() error {
err := k8sClient.Get(ctx, client.ObjectKey{Namespace: ns, Name: appName}, &testApp)
if err != nil {
return err
}
if testApp.Status.Phase != oamcomm.ApplicationWorkflowSuspending {
return fmt.Errorf("application status wants %s, actually %s", oamcomm.ApplicationWorkflowSuspending, testApp.Status.Phase)
if testApp.Status.Phase != oamcomm.ApplicationWorkflowTerminated {
return fmt.Errorf("application status wants %s, actually %s", oamcomm.ApplicationWorkflowTerminated, testApp.Status.Phase)
}
return nil
}, 120*time.Second, time.Second).Should(BeNil())
@@ -335,7 +335,7 @@ var _ = Describe("Application Normal tests", func() {
Expect(k8sClient.Create(ctx, &newApp)).Should(BeNil())
By("check application status")
verifyApplicationWorkflowSuspending(newApp.Namespace, newApp.Name)
verifyApplicationWorkflowTerminated(newApp.Namespace, newApp.Name)
})
It("Test wait suspend", func() {
@@ -425,7 +425,7 @@ var _ = Describe("Application Normal tests", func() {
Expect(k8sClient.Create(ctx, &newApp)).Should(BeNil())
By("Checking an application status")
verifyApplicationWorkflowSuspending(newApp.Namespace, newApp.Name)
verifyApplicationWorkflowTerminated(newApp.Namespace, newApp.Name)
})
It("Test app with non-existence ServiceAccount", func() {
@@ -453,6 +453,6 @@ var _ = Describe("Application Normal tests", func() {
Expect(k8sClient.Create(ctx, &newApp)).Should(BeNil())
By("Checking an application status")
verifyApplicationWorkflowSuspending(newApp.Namespace, newApp.Name)
verifyApplicationWorkflowTerminated(newApp.Namespace, newApp.Name)
})
})