Refactor application code to make it run as Dag workflow (#2236)

* Refactor: remove use of AppConfig in AppRevision

* Refactor: remove insert secret and configmap

* Feat(workflow): upgrade

* Fix(conflict): workflow cherry

* Feat(workflow): support DAG mode

* Feat(workflow): prepare steps in step

* Feat(tools): random string

* Fix(rand): gosec weak random

* Fix(ci): test passing

* Feat(workflow): generate steps

* Fix: fix rebase from master

* Fix: fix workflow ut

* Feat(test): add test cases

* Fix: fix lint and rebase from master

* Refactor: application code

* Fix: fix ci lint

* Fix: make code reviewable

* Fix: workflow_test.go

* Feat: collect services

* Fix(ci): unit tests

* Feat: make one

* Test: application with input/output and workflow

* Fix: trace test

* Fix: update step index falied

* Feat: refactor op.#Load

* Fix: delete dead code

* Refactor: op.xxx

* Fix: patch component

* Test: add generator test

* Fix: add license

* Fix: pending e2e plugin test

* Fix: disable test/e2e-test

* Fix: patch by script

Co-authored-by: 天元 <jianbo.sjb@alibaba-inc.com>
Co-authored-by: FogDong <dongtianxin.tx@alibaba-inc.com>

references/appfile/dryrun/diff.go

@@ -29,15 +29,16 @@ import (
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
 	"github.com/oam-dev/kubevela/apis/types"
+	"github.com/oam-dev/kubevela/pkg/appfile"
 	"github.com/oam-dev/kubevela/pkg/cue/packages"
 	"github.com/oam-dev/kubevela/pkg/oam"
 	"github.com/oam-dev/kubevela/pkg/oam/discoverymapper"
-	"github.com/oam-dev/kubevela/pkg/oam/util"
 )
 
 // NewLiveDiffOption creates a live-diff option
 func NewLiveDiffOption(c client.Client, dm discoverymapper.DiscoveryMapper, pd *packages.PackageDiscover, as []oam.Object) *LiveDiffOption {
-	return &LiveDiffOption{NewDryRunOption(c, dm, pd, as)}
+	parser := appfile.NewApplicationParser(c, dm, pd)
+	return &LiveDiffOption{DryRun: NewDryRunOption(c, dm, pd, as), Parser: parser}
 }
 
 // ManifestKind enums the kind of OAM objects
@@ -87,6 +88,7 @@ type manifest struct {
 // living AppRevision in the cluster
 type LiveDiffOption struct {
 	DryRun
+	Parser *appfile.Parser
 }
 
 // Diff does three phases, dry-run on input app, preparing manifest for diff, and
@@ -103,7 +105,7 @@ func (l *LiveDiffOption) Diff(ctx context.Context, app *v1beta1.Application, app
 	}
 
 	// old refers to the living app revision
-	oldManifest, err := generateManifestFromAppRevision(appRevision)
+	oldManifest, err := generateManifestFromAppRevision(l.Parser, appRevision)
 	if err != nil {
 		return nil, errors.WithMessagef(err, "cannot generate diff manifest for AppRevision %q", appRevision.Name)
 	}
@@ -315,9 +317,12 @@ func generateManifest(app *v1beta1.Application, comps []*types.ComponentManifest
 }
 
 // generateManifestFromAppRevision generates manifest from an AppRevision
-func generateManifestFromAppRevision(appRevision *v1beta1.ApplicationRevision) (*manifest, error) {
-	// comps, err := util.ConvertRawComponentManifests(appRevision.Spec.ComponentManifests)
-	comps, err := util.AppConfig2ComponentManifests(appRevision.Spec.ApplicationConfiguration, appRevision.Spec.Components)
+func generateManifestFromAppRevision(parser *appfile.Parser, appRevision *v1beta1.ApplicationRevision) (*manifest, error) {
+	af, err := parser.GenerateAppFileFromRevision(appRevision)
+	if err != nil {
+		return nil, err
+	}
+	comps, err := af.GenerateComponentManifests()
 	if err != nil {
 		return nil, err
 	}

references/appfile/dryrun/dryrun.go

@@ -72,26 +72,5 @@ func (d *Option) ExecuteDryRun(ctx context.Context, app *v1beta1.Application) ([
 		return nil, errors.WithMessage(err, "cannot generate AppConfig and Components")
 	}
 
-	for _, comp := range comps {
-		if comp.StandardWorkload != nil {
-			if comp.StandardWorkload.GetName() == "" {
-				comp.StandardWorkload.SetName(comp.Name)
-			}
-			if comp.StandardWorkload.GetNamespace() == "" {
-				comp.StandardWorkload.SetNamespace(appFile.Namespace)
-			}
-		}
-
-		for _, trait := range comp.Traits {
-			if trait.GetName() == "" {
-				traitType := trait.GetLabels()[oam.TraitTypeLabel]
-				traitName := oamutil.GenTraitNameCompatible(comp.Name, trait, traitType)
-				trait.SetName(traitName)
-			}
-			if trait.GetNamespace() == "" {
-				trait.SetNamespace(appFile.Namespace)
-			}
-		}
-	}
 	return comps, nil
 }

references/appfile/dryrun/dryrun_test.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/oam-dev/kubevela/apis/types"
+
 	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -46,9 +48,10 @@ var _ = Describe("Test DryRun", func() {
 		expectCompYAML := readDataFromFile("./testdata/dryrun-exp-comp.yaml")
 		By("Verify generated Comp")
 		Expect(comps).ShouldNot(BeEmpty())
-		resultCompStr, err := yaml.Marshal(comps[0])
+		var expC = types.ComponentManifest{}
+		err = yaml.Unmarshal([]byte(expectCompYAML), &expC)
 		Expect(err).Should(BeNil())
-		diff := cmp.Diff(expectCompYAML, string(resultCompStr))
+		diff := cmp.Diff(&expC, comps[0])
 		Expect(diff).Should(BeEmpty())
 	})
 })

references/appfile/dryrun/suit_test.go

@@ -22,6 +22,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/oam-dev/kubevela/pkg/appfile"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
@@ -106,7 +108,7 @@ var _ = BeforeSuite(func(done Done) {
 	Expect(err).Should(BeNil())
 
 	dryrunOpt = NewDryRunOption(k8sClient, dm, pd, []oam.Object{cdMyWorker, tdMyIngress, tdMyScaler})
-	diffOpt = &LiveDiffOption{dryrunOpt}
+	diffOpt = &LiveDiffOption{DryRun: dryrunOpt, Parser: appfile.NewApplicationParser(k8sClient, dm, pd)}
 
 	close(done)
 }, 60)

references/appfile/dryrun/testdata/diff-apprevision.yaml

@@ -1,8 +1,6 @@
 apiVersion: core.oam.dev/v1beta1
 kind: ApplicationRevision
 metadata:
-  annotations:
-    oam.dev/kubevela-version: undefined
   generation: 1
   labels:
     app.oam.dev/app-revision-hash: c2207054644744d
@@ -109,7 +107,7 @@ spec:
                 app.oam.dev/name: livediff-demo
                 trait.oam.dev/resource: scaler
                 trait.oam.dev/type: myscaler
-              name: myweb-1-myscaler-5547bdc57f
+              name: myweb-1-myscaler-69ccc5855d
               namespace: default
             spec:
               replicaCount: 2

references/appfile/dryrun/testdata/dryrun-exp-comp.yaml

@@ -1,5 +1,4 @@
 ExternalRevision: ""
-InsertConfigNotReady: false
 Name: myweb
 Namespace: default
 PackagedTraitResources: null
@@ -11,11 +10,13 @@ StandardWorkload:
   apiVersion: apps/v1
   kind: Deployment
   metadata:
+    annotations: {}
     labels:
       app.oam.dev/appRevision: ""
       app.oam.dev/component: myweb
       app.oam.dev/name: app-dryrun
       workload.oam.dev/type: myworker
+      app.oam.dev/resourceType: WORKLOAD
     name: myweb
     namespace: default
   spec:
@@ -37,12 +38,14 @@ Traits:
 - apiVersion: v1
   kind: Service
   metadata:
+    annotations: {}
     labels:
       app.oam.dev/appRevision: ""
       app.oam.dev/component: myweb
       app.oam.dev/name: app-dryrun
       trait.oam.dev/resource: service
       trait.oam.dev/type: myingress
+      app.oam.dev/resourceType: TRAIT
     name: myweb
     namespace: default
   spec:
@@ -54,12 +57,14 @@ Traits:
 - apiVersion: networking.k8s.io/v1beta1
   kind: Ingress
   metadata:
+    annotations: {}
     labels:
       app.oam.dev/appRevision: ""
       app.oam.dev/component: myweb
       app.oam.dev/name: app-dryrun
       trait.oam.dev/resource: ingress
       trait.oam.dev/type: myingress
+      app.oam.dev/resourceType: TRAIT
     name: myweb
     namespace: default
   spec: