[Backport release-1.1] Feat: store workflow step def properties in cm (#2614)

* Fix: fix notification def

(cherry picked from commit f35a213048)

* Feat: store workflow step def properties in cm

(cherry picked from commit 7f3902536c)

* fix ci

(cherry picked from commit a252749f77)

* fix data race

(cherry picked from commit 0b55ce8386)

Co-authored-by: FogDong <dongtianxin.tx@alibaba-inc.com>

apis/core.oam.dev/v1beta1/workflow_step_definition.go

@@ -38,7 +38,8 @@ type WorkflowStepDefinitionSpec struct {
 type WorkflowStepDefinitionStatus struct {
 	// ConditionedStatus reflects the observed status of a resource
 	condition.ConditionedStatus `json:",inline"`
-
+	// ConfigMapRef refer to a ConfigMap which contains OpenAPI V3 JSON schema of Component parameters.
+	ConfigMapRef string `json:"configMapRef,omitempty"`
 	// LatestRevision of the component definition
 	// +optional
 	LatestRevision *common.Revision `json:"latestRevision,omitempty"`

charts/vela-core/crds/core.oam.dev_applicationrevisions.yaml

@@ -4399,6 +4399,10 @@ spec:
                             - type
                             type: object
                           type: array
+                        configMapRef:
+                          description: ConfigMapRef refer to a ConfigMap which contains
+                            OpenAPI V3 JSON schema of Component parameters.
+                          type: string
                         latestRevision:
                           description: LatestRevision of the component definition
                           properties:

charts/vela-core/crds/core.oam.dev_definitionrevisions.yaml

@@ -1123,6 +1123,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      configMapRef:
+                        description: ConfigMapRef refer to a ConfigMap which contains
+                          OpenAPI V3 JSON schema of Component parameters.
+                        type: string
                       latestRevision:
                         description: LatestRevision of the component definition
                         properties:

charts/vela-core/crds/core.oam.dev_workflowstepdefinitions.yaml

@@ -216,6 +216,10 @@ spec:
                   - type
                   type: object
                 type: array
+              configMapRef:
+                description: ConfigMapRef refer to a ConfigMap which contains OpenAPI
+                  V3 JSON schema of Component parameters.
+                type: string
               latestRevision:
                 description: LatestRevision of the component definition
                 properties:

charts/vela-core/templates/defwithtemplate/webhook-notification.yaml

@@ -97,17 +97,17 @@ spec:
         		url?:       string
         		value?:     string
         		style?:     string
-        		text?:      text
+        		text?:      textType
         		confirm?: {
-        			title:   text
-        			text:    text
-        			confirm: text
-        			deny:    text
+        			title:   textType
+        			text:    textType
+        			confirm: textType
+        			deny:    textType
         			style?:  string
         		}
         		options?: [...option]
         		initial_options?: [...option]
-        		placeholder?:  text
+        		placeholder?:  textType
         		initial_date?: string
         		image_url?:    string
         		alt_text?:     string
@@ -121,16 +121,16 @@ spec:
         		initial_time?: string
         	}]
         }
-        text: {
+        textType: {
         	type:      string
         	text:      string
         	emoji?:    bool
         	verbatim?: bool
         }
         option: {
-        	text:         text
+        	text:         textType
         	value:        string
-        	description?: text
+        	description?: textType
         	url?:         string
         }
         // send webhook notification

charts/vela-minimal/crds/core.oam.dev_applicationrevisions.yaml

@@ -4399,6 +4399,10 @@ spec:
                             - type
                             type: object
                           type: array
+                        configMapRef:
+                          description: ConfigMapRef refer to a ConfigMap which contains
+                            OpenAPI V3 JSON schema of Component parameters.
+                          type: string
                         latestRevision:
                           description: LatestRevision of the component definition
                           properties:

charts/vela-minimal/crds/core.oam.dev_definitionrevisions.yaml

@@ -1123,6 +1123,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      configMapRef:
+                        description: ConfigMapRef refer to a ConfigMap which contains
+                          OpenAPI V3 JSON schema of Component parameters.
+                        type: string
                       latestRevision:
                         description: LatestRevision of the component definition
                         properties:

charts/vela-minimal/crds/core.oam.dev_workflowstepdefinitions.yaml

@@ -216,6 +216,10 @@ spec:
                   - type
                   type: object
                 type: array
+              configMapRef:
+                description: ConfigMapRef refer to a ConfigMap which contains OpenAPI
+                  V3 JSON schema of Component parameters.
+                type: string
               latestRevision:
                 description: LatestRevision of the component definition
                 properties:

charts/vela-minimal/templates/defwithtemplate/webhook-notification.yaml

@@ -97,17 +97,17 @@ spec:
         		url?:       string
         		value?:     string
         		style?:     string
-        		text?:      text
+        		text?:      textType
         		confirm?: {
-        			title:   text
-        			text:    text
-        			confirm: text
-        			deny:    text
+        			title:   textType
+        			text:    textType
+        			confirm: textType
+        			deny:    textType
         			style?:  string
         		}
         		options?: [...option]
         		initial_options?: [...option]
-        		placeholder?:  text
+        		placeholder?:  textType
         		initial_date?: string
         		image_url?:    string
         		alt_text?:     string
@@ -121,16 +121,16 @@ spec:
         		initial_time?: string
         	}]
         }
-        text: {
+        textType: {
         	type:      string
         	text:      string
         	emoji?:    bool
         	verbatim?: bool
         }
         option: {
-        	text:         text
+        	text:         textType
         	value:        string
-        	description?: text
+        	description?: textType
         	url?:         string
         }
         // send webhook notification

legacy/charts/vela-core-legacy/crds/core.oam.dev_applicationrevisions.yaml

@@ -4399,6 +4399,10 @@ spec:
                             - type
                             type: object
                           type: array
+                        configMapRef:
+                          description: ConfigMapRef refer to a ConfigMap which contains
+                            OpenAPI V3 JSON schema of Component parameters.
+                          type: string
                         latestRevision:
                           description: LatestRevision of the component definition
                           properties:

legacy/charts/vela-core-legacy/crds/core.oam.dev_definitionrevisions.yaml

@@ -1123,6 +1123,10 @@ spec:
                           - type
                           type: object
                         type: array
+                      configMapRef:
+                        description: ConfigMapRef refer to a ConfigMap which contains
+                          OpenAPI V3 JSON schema of Component parameters.
+                        type: string
                       latestRevision:
                         description: LatestRevision of the component definition
                         properties:

legacy/charts/vela-core-legacy/crds/core.oam.dev_workflowstepdefinitions.yaml

@@ -216,6 +216,10 @@ spec:
                   - type
                   type: object
                 type: array
+              configMapRef:
+                description: ConfigMapRef refer to a ConfigMap which contains OpenAPI
+                  V3 JSON schema of Component parameters.
+                type: string
               latestRevision:
                 description: LatestRevision of the component definition
                 properties:

pkg/controller/core.oam.dev/v1alpha2/core/workflow/workflowstepdefinition/workflowstepdefinition_controller.go

@@ -122,6 +122,28 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		r.record.Event(&wfstepdefinition, event.Warning("failed to garbage collect DefinitionRevision of type WorkflowStepDefinition", err))
 	}
 
+	def := utils.NewCapabilityStepDef(&wfstepdefinition)
+	def.Name = req.NamespacedName.Name
+	// Store the parameter of stepDefinition to configMap
+	cmName, err := def.StoreOpenAPISchema(ctx, r.Client, r.pd, req.Namespace, req.Name, defRev.Name)
+	if err != nil {
+		klog.InfoS("Could not store capability in ConfigMap", "err", err)
+		r.record.Event(&(wfstepdefinition), event.Warning("Could not store capability in ConfigMap", err))
+		return ctrl.Result{}, util.PatchCondition(ctx, r, &wfstepdefinition,
+			condition.ReconcileError(fmt.Errorf(util.ErrStoreCapabilityInConfigMap, wfstepdefinition.Name, err)))
+	}
+
+	if wfstepdefinition.Status.ConfigMapRef != cmName {
+		wfstepdefinition.Status.ConfigMapRef = cmName
+		if err := r.UpdateStatus(ctx, &wfstepdefinition); err != nil {
+			klog.ErrorS(err, "Could not update WorkflowStepDefinition Status", "workflowStepDefinition", klog.KRef(req.Namespace, req.Name))
+			r.record.Event(&wfstepdefinition, event.Warning("Could not update WorkflowStepDefinition Status", err))
+			return ctrl.Result{}, util.PatchCondition(ctx, r, &wfstepdefinition,
+				condition.ReconcileError(fmt.Errorf(util.ErrUpdateWorkflowStepDefinition, wfstepdefinition.Name, err)))
+		}
+		klog.InfoS("Successfully updated the status.configMapRef of the WorkflowStepDefinition", "workflowStepDefinition",
+			klog.KRef(req.Namespace, req.Name), "status.configMapRef", cmName)
+	}
 	return ctrl.Result{}, nil
 }
 

pkg/controller/utils/capability.go

@@ -351,6 +351,75 @@ func (def *CapabilityTraitDefinition) StoreOpenAPISchema(ctx context.Context, k8
 	return cmName, nil
 }
 
+// CapabilityStepDefinition is the Capability struct for WorkflowStepDefinition
+type CapabilityStepDefinition struct {
+	Name           string                         `json:"name"`
+	StepDefinition v1beta1.WorkflowStepDefinition `json:"stepDefinition"`
+
+	CapabilityBaseDefinition
+}
+
+// NewCapabilityStepDef will create a CapabilityStepDefinition
+func NewCapabilityStepDef(stepdefinition *v1beta1.WorkflowStepDefinition) CapabilityStepDefinition {
+	var def CapabilityStepDefinition
+	def.Name = stepdefinition.Name
+	def.StepDefinition = *stepdefinition.DeepCopy()
+	return def
+}
+
+// GetOpenAPISchema gets OpenAPI v3 schema by StepDefinition name
+func (def *CapabilityStepDefinition) GetOpenAPISchema(pd *packages.PackageDiscover, name string) ([]byte, error) {
+	capability, err := appfile.ConvertTemplateJSON2Object(name, nil, def.StepDefinition.Spec.Schematic)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert WorkflowStepDefinition to Capability Object")
+	}
+	return getOpenAPISchema(capability, pd)
+}
+
+// StoreOpenAPISchema stores OpenAPI v3 schema from StepDefinition in ConfigMap
+func (def *CapabilityStepDefinition) StoreOpenAPISchema(ctx context.Context, k8sClient client.Client, pd *packages.PackageDiscover, namespace, name string, revName string) (string, error) {
+	var jsonSchema []byte
+	var err error
+
+	jsonSchema, err = def.GetOpenAPISchema(pd, name)
+	if err != nil {
+		return "", fmt.Errorf("failed to generate OpenAPI v3 JSON schema for capability %s: %w", def.Name, err)
+	}
+
+	stepDefinition := def.StepDefinition
+	ownerReference := []metav1.OwnerReference{{
+		APIVersion:         stepDefinition.APIVersion,
+		Kind:               stepDefinition.Kind,
+		Name:               stepDefinition.Name,
+		UID:                stepDefinition.GetUID(),
+		Controller:         pointer.BoolPtr(true),
+		BlockOwnerDeletion: pointer.BoolPtr(true),
+	}}
+	cmName, err := def.CreateOrUpdateConfigMap(ctx, k8sClient, namespace, stepDefinition.Name, jsonSchema, ownerReference)
+	if err != nil {
+		return cmName, err
+	}
+
+	// Create a configmap to store parameter for each definitionRevision
+	defRev := new(v1beta1.DefinitionRevision)
+	if err = k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: revName}, defRev); err != nil {
+		return "", err
+	}
+	ownerReference = []metav1.OwnerReference{{
+		APIVersion:         defRev.APIVersion,
+		Kind:               defRev.Kind,
+		Name:               defRev.Name,
+		UID:                defRev.GetUID(),
+		Controller:         pointer.BoolPtr(true),
+		BlockOwnerDeletion: pointer.BoolPtr(true),
+	}}
+	_, err = def.CreateOrUpdateConfigMap(ctx, k8sClient, namespace, revName, jsonSchema, ownerReference)
+	if err != nil {
+		return cmName, err
+	}
+	return cmName, nil
+}
+
 // CapabilityBaseDefinition is the base struct for CapabilityWorkloadDefinition and CapabilityTraitDefinition
 type CapabilityBaseDefinition struct {
 }
@@ -399,7 +468,7 @@ func (def *CapabilityBaseDefinition) CreateOrUpdateConfigMap(ctx context.Context
 	return cmName, nil
 }
 
-// getDefinition is the main function for GetDefinition API
+// getOpenAPISchema is the main function for GetDefinition API
 func getOpenAPISchema(capability types.Capability, pd *packages.PackageDiscover) ([]byte, error) {
 	openAPISchema, err := generateOpenAPISchemaFromCapabilityParameter(capability, pd)
 	if err != nil {

pkg/cue/packages/package.go

@@ -35,6 +35,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+
+	"github.com/oam-dev/kubevela/pkg/stdlib"
 )
 
 const (
@@ -107,6 +109,9 @@ func (pd *PackageDiscover) ImportBuiltinPackagesFor(bi *build.Instance) {
 // ImportPackagesAndBuildInstance Combine import built-in packages and build cue template together to avoid data race
 func (pd *PackageDiscover) ImportPackagesAndBuildInstance(bi *build.Instance) (inst *cue.Instance, err error) {
 	pd.ImportBuiltinPackagesFor(bi)
+	if err := stdlib.AddImportsFor(bi, ""); err != nil {
+		return nil, err
+	}
 	var r cue.Runtime
 	pd.mutex.Lock()
 	defer pd.mutex.Unlock()

pkg/oam/util/helper.go

@@ -100,6 +100,8 @@ const (
 	ErrUpdateComponentDefinition = "cannot update ComponentDefinition %s: %v"
 	// ErrUpdateTraitDefinition is the error while update TraitDefinition
 	ErrUpdateTraitDefinition = "cannot update TraitDefinition %s: %v"
+	// ErrUpdateStepDefinition is the error while update WorkflowStepDefinition
+	ErrUpdateStepDefinition = "cannot update WorkflowStepDefinition %s: %v"
 	// ErrUpdatePolicyDefinition is the error while update PolicyDefinition
 	ErrUpdatePolicyDefinition = "cannot update PolicyDefinition %s: %v"
 	// ErrUpdateWorkflowStepDefinition is the error while update WorkflowStepDefinition

pkg/stdlib/packages.go

@@ -27,8 +27,7 @@ import (
 
 var (
 	//go:embed pkgs op.cue
-	fs         embed.FS
-	pkgContent string
+	fs embed.FS
 )
 
 // GetPackages Get Stdlib packages
@@ -44,7 +43,7 @@ func GetPackages(tagTempl string) (map[string]string, error) {
 		return nil, err
 	}
 
-	pkgContent = string(opBytes) + "\n"
+	pkgContent := string(opBytes) + "\n"
 	for _, file := range files {
 		body, err := fs.ReadFile("pkgs/" + file.Name())
 		if err != nil {

pkg/stdlib/pkgs/slack.cue

@@ -18,17 +18,17 @@
 		url?:       string
 		value?:     string
 		style?:     string
-		text?:      #text
+		text?:      #textType
 		confirm?: {
-			title:   #text
-			text:    #text
-			confirm: #text
-			deny:    #text
+			title:   #textType
+			text:    #textType
+			confirm: #textType
+			deny:    #textType
 			style?:  string
 		}
 		options?: [...#option]
 		initial_options?: [...#option]
-		placeholder?:  #text
+		placeholder?:  #textType
 		initial_date?: string
 		image_url?:    string
 		alt_text?:     string
@@ -45,7 +45,7 @@
 	}]
 }
 
-#text: {
+#textType: {
 	type:      string
 	text:      string
 	emoji?:    bool
@@ -53,8 +53,8 @@
 }
 
 #option: {
-	text:         text
+	text:         #textType
 	value:        string
-	description?: text
+	description?: #textType
 	url?:         string
 }

vela-templates/definitions/internal/webhook-notification.cue

@@ -93,17 +93,17 @@ template: {
 			url?:       string
 			value?:     string
 			style?:     string
-			text?:      text
+			text?:      textType
 			confirm?: {
-				title:   text
-				text:    text
-				confirm: text
-				deny:    text
+				title:   textType
+				text:    textType
+				confirm: textType
+				deny:    textType
 				style?:  string
 			}
 			options?: [...option]
 			initial_options?: [...option]
-			placeholder?:  text
+			placeholder?:  textType
 			initial_date?: string
 			image_url?:    string
 			alt_text?:     string
@@ -120,7 +120,7 @@ template: {
 		}]
 	}
 
-	text: {
+	textType: {
 		type:      string
 		text:      string
 		emoji?:    bool
@@ -128,9 +128,9 @@ template: {
 	}
 
 	option: {
-		text:         text
+		text:         textType
 		value:        string
-		description?: text
+		description?: textType
 		url?:         string
 	}