Generate OpenAPI JSON schema for Terraform Component (#1738) (#1753)

Fix #1736

pkg/appfile/template.go

@@ -242,7 +242,7 @@ func loadSchematicToTemplate(tmpl *Template, status *common.Status, schematic *c
 	return nil
 }
 
-// ConvertTemplateJSON2Object convert spec.extension to object
+// ConvertTemplateJSON2Object convert spec.extension or spec.schematic to object
 func ConvertTemplateJSON2Object(capabilityName string, in *runtime.RawExtension, schematic *common.Schematic) (types.Capability, error) {
 	var t types.Capability
 	t.Name = capabilityName

pkg/controller/core.oam.dev/v1alpha2/core/components/componentdefinition/componentdefinition_controller.go

@@ -138,6 +138,8 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		def.Helm = componentDefinition.Spec.Schematic.HELM
 	case util.KubeDef:
 		def.Kube = componentDefinition.Spec.Schematic.KUBE
+	case util.TerraformDef:
+		def.Terraform = componentDefinition.Spec.Schematic.Terraform
 	default:
 	}
 

pkg/controller/core.oam.dev/v1alpha2/core/components/componentdefinition/componentdefinition_controller_test.go

@@ -275,6 +275,115 @@ spec:
 		})
 	})
 
+	Context("When the ComponentDefinition contains Terraform Module, should create a ConfigMap", func() {
+		var componentDefinitionName = "alibaba-rds-test"
+		var namespace = "default"
+		req := reconcile.Request{NamespacedName: client.ObjectKey{Name: componentDefinitionName, Namespace: namespace}}
+
+		It("Applying Terraform ComponentDefinition", func() {
+			By("Apply ComponentDefinition")
+			var validComponentDefinition = `
+apiVersion: core.oam.dev/v1alpha2
+kind: ComponentDefinition
+metadata:
+  name: alibaba-rds-test
+  annotations:
+    definition.oam.dev/description: Terraform configuration for Alibaba Cloud RDS object
+    type: terraform
+spec:
+  workload:
+    definition:
+      apiVersion: apps/v1
+      kind: Deployment
+  schematic:
+    terraform:
+      configuration: |
+        module "rds" {
+          source = "terraform-alicloud-modules/rds/alicloud"
+          engine = "MySQL"
+          engine_version = "8.0"
+          instance_type = "rds.mysql.c1.large"
+          instance_storage = "20"
+          instance_name = var.instance_name
+          account_name = var.account_name
+          password = var.password
+        }
+
+        output "DB_NAME" {
+          value = module.rds.this_db_instance_name
+        }
+        output "DB_USER" {
+          value = module.rds.this_db_database_account
+        }
+        output "DB_PORT" {
+          value = module.rds.this_db_instance_port
+        }
+        output "DB_HOST" {
+          value = module.rds.this_db_instance_connection_string
+        }
+        output "DB_PASSWORD" {
+          value = module.rds.this_db_instance_port
+        }
+
+        variable "instance_name" {
+          description = "RDS instance name"
+          type = string
+          default = "poc"
+        }
+
+        variable "account_name" {
+          description = "RDS instance user account name"
+          type = "string"
+          default = "oam"
+        }
+
+        variable "password" {
+          description = "RDS instance account password"
+          type = "string"
+          default = "xxx"
+        }
+
+        variable "intVar" {
+          type = "number"
+        }
+
+        variable "boolVar" {
+          type = "bool"
+        }
+
+        variable "listVar" {
+          type = "list"
+        }
+
+        variable "mapVar" {
+          type = "map"
+        }
+
+`
+
+			var def v1beta1.ComponentDefinition
+			Expect(yaml.Unmarshal([]byte(validComponentDefinition), &def)).Should(BeNil())
+			def.Namespace = namespace
+			Expect(k8sClient.Create(ctx, &def)).Should(Succeed())
+			reconcileRetry(&r, req)
+
+			By("Check whether ConfigMap is created")
+			var cm corev1.ConfigMap
+			name := fmt.Sprintf("%s%s", types.CapabilityConfigMapNamePrefix, componentDefinitionName)
+			Eventually(func() bool {
+				err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &cm)
+				return err == nil
+			}, 10*time.Second, time.Second).Should(BeTrue())
+			Expect(cm.Data[types.OpenapiV3JSONSchema]).Should(Not(Equal("")))
+
+			By("Check whether ConfigMapRef reference to the right ComponentDefinition")
+			Eventually(func() string {
+				_ = k8sClient.Get(ctx, client.ObjectKey{Namespace: def.Namespace, Name: def.Name}, &def)
+				return def.Status.ConfigMapRef
+			}, 10*time.Second, time.Second).Should(Equal(name))
+		})
+	})
+
 	Context("When the ComponentDefinition is invalid, should hit issues", func() {
 		var namespace = "ns-def"
 		BeforeEach(func() {

pkg/controller/core.oam.dev/v1alpha2/core/components/componentdefinition/handler.go

@@ -44,11 +44,17 @@ func (h *handler) CreateWorkloadDefinition(ctx context.Context) (util.WorkloadTy
 		workloadType = util.ReferWorkload
 		workloadName = h.cd.Spec.Workload.Type
 	}
-	if h.cd.Spec.Schematic != nil && h.cd.Spec.Schematic.HELM != nil {
-		workloadType = util.HELMDef
-	}
-	if h.cd.Spec.Schematic != nil && h.cd.Spec.Schematic.KUBE != nil {
-		workloadType = util.KubeDef
+
+	if h.cd.Spec.Schematic != nil {
+		if h.cd.Spec.Schematic.HELM != nil {
+			workloadType = util.HELMDef
+		}
+		if h.cd.Spec.Schematic.KUBE != nil {
+			workloadType = util.KubeDef
+		}
+		if h.cd.Spec.Schematic.Terraform != nil {
+			workloadType = util.TerraformDef
+		}
 	}
 
 	wd := new(v1beta1.WorkloadDefinition)

pkg/controller/utils/capability.go

@@ -48,6 +48,17 @@ import (
 // ErrNoSectionParameterInCue means there is not parameter section in Cue template of a workload
 const ErrNoSectionParameterInCue = "capability %s doesn't contain section `parameter`"
 
+// data types of parameter value
+const (
+	TerraformVariableString string = "string"
+	TerraformVariableNumber string = "number"
+	TerraformVariableBool   string = "bool"
+	TerraformVariableList   string = "list"
+	TerraformVariableTuple  string = "tuple"
+	TerraformVariableMap    string = "map"
+	TerraformVariableObject string = "object"
+)
+
 // CapabilityDefinitionInterface is the interface for Capability (WorkloadDefinition and TraitDefinition)
 type CapabilityDefinitionInterface interface {
 	GetCapabilityObject(ctx context.Context, k8sClient client.Client, namespace, name string) (*types.Capability, error)
@@ -62,8 +73,9 @@ type CapabilityComponentDefinition struct {
 	WorkloadType    util.WorkloadType `json:"workloadType"`
 	WorkloadDefName string            `json:"workloadDefName"`
 
-	Helm *commontypes.Helm `json:"helm"`
-	Kube *commontypes.Kube `json:"kube"`
+	Helm      *commontypes.Helm      `json:"helm"`
+	Kube      *commontypes.Kube      `json:"kube"`
+	Terraform *commontypes.Terraform `json:"terraform"`
 	CapabilityBaseDefinition
 }
 
@@ -133,7 +145,44 @@ func (def *CapabilityComponentDefinition) GetKubeSchematicOpenAPISchema(params [
 		}
 		properties[p.Name] = tmp
 	}
-	s := openapi3.NewObjectSchema().WithProperties(properties)
+	return generateJSONSchemaWithRequiredProperty(properties, required)
+}
+
+// GetOpenAPISchemaFromTerraformComponentDefinition gets OpenAPI v3 schema by WorkloadDefinition name
+func GetOpenAPISchemaFromTerraformComponentDefinition(configuration string) ([]byte, error) {
+	schemas := make(map[string]*openapi3.Schema)
+	var required []string
+	variables, err := common.ParseTerraformVariables(configuration)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate capability properties")
+	}
+	for k, v := range variables {
+		var schema *openapi3.Schema
+		switch v.Type {
+		case TerraformVariableString:
+			schema = openapi3.NewStringSchema()
+		case TerraformVariableNumber:
+			schema = openapi3.NewFloat64Schema()
+		case TerraformVariableBool:
+			schema = openapi3.NewBoolSchema()
+		case TerraformVariableList, TerraformVariableTuple:
+			schema = openapi3.NewArraySchema()
+		case TerraformVariableMap, TerraformVariableObject:
+			schema = openapi3.NewObjectSchema()
+		}
+		schema.Title = k
+		required = append(required, k)
+		if v.Default != nil {
+			schema.Default = v.Default
+		}
+		schema.Description = v.Description
+		schemas[v.Name] = schema
+	}
+	return generateJSONSchemaWithRequiredProperty(schemas, required)
+}
+
+func generateJSONSchemaWithRequiredProperty(schemas map[string]*openapi3.Schema, required []string) ([]byte, error) {
+	s := openapi3.NewObjectSchema().WithProperties(schemas)
 	if len(required) > 0 {
 		s.Required = required
 	}
@@ -154,6 +203,11 @@ func (def *CapabilityComponentDefinition) StoreOpenAPISchema(ctx context.Context
 		jsonSchema, err = helm.GetChartValuesJSONSchema(ctx, def.Helm)
 	case util.KubeDef:
 		jsonSchema, err = def.GetKubeSchematicOpenAPISchema(def.Kube.Parameters)
+	case util.TerraformDef:
+		if def.Terraform == nil {
+			return fmt.Errorf("no Configuration is set in Terraform specification: %s", def.Name)
+		}
+		jsonSchema, err = GetOpenAPISchemaFromTerraformComponentDefinition(def.Terraform.Configuration)
 	default:
 		jsonSchema, err = def.GetOpenAPISchema(ctx, k8sClient, pd, namespace, name)
 	}

pkg/controller/utils/capability_test.go

@@ -22,6 +22,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/crossplane/crossplane-runtime/pkg/test"
@@ -148,3 +149,73 @@ func TestGenerateOpenAPISchemaFromCapabilityParameter(t *testing.T) {
 		})
 	}
 }
+
+func TestGetOpenAPISchemaFromTerraformComponentDefinition(t *testing.T) {
+	configuration := `
+module "rds" {
+  source = "terraform-alicloud-modules/rds/alicloud"
+  engine = "MySQL"
+  engine_version = "8.0"
+  instance_type = "rds.mysql.c1.large"
+  instance_storage = "20"
+  instance_name = var.instance_name
+  account_name = var.account_name
+  password = var.password
+}
+
+output "DB_NAME" {
+  value = module.rds.this_db_instance_name
+}
+output "DB_USER" {
+  value = module.rds.this_db_database_account
+}
+output "DB_PORT" {
+  value = module.rds.this_db_instance_port
+}
+output "DB_HOST" {
+  value = module.rds.this_db_instance_connection_string
+}
+output "DB_PASSWORD" {
+  value = module.rds.this_db_instance_port
+}
+
+variable "instance_name" {
+  description = "RDS instance name"
+  type = string
+  default = "poc"
+}
+
+variable "account_name" {
+  description = "RDS instance user account name"
+  type = "string"
+  default = "oam"
+}
+
+variable "password" {
+  description = "RDS instance account password"
+  type = "string"
+  default = "xxx"
+}
+
+variable "intVar" {
+  type = "number"
+}
+
+variable "boolVar" {
+  type = "bool"
+}
+
+variable "listVar" {
+  type = "list"
+}
+
+variable "mapVar" {
+  type = "map"
+}`
+
+	schema, err := GetOpenAPISchemaFromTerraformComponentDefinition(configuration)
+	assert.NilError(t, err)
+	data := string(schema)
+	assert.Equal(t, strings.Contains(data, "account_name"), true)
+	assert.Equal(t, strings.Contains(data, "intVar"), true)
+}

pkg/oam/util/helper.go

@@ -127,6 +127,9 @@ const (
 	// HELMDef describe a workload refer to HELM
 	HELMDef WorkloadType = "HelmDef"
 
+	// TerraformDef describes a workload refer to Terraform
+	TerraformDef WorkloadType = "TerraformDef"
+
 	// ReferWorkload describe an existing workload
 	ReferWorkload WorkloadType = "ReferWorkload"
 )

pkg/utils/common/common.go

@@ -33,6 +33,8 @@ import (
 	"cuelang.org/go/encoding/openapi"
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/ghodss/yaml"
+	"github.com/hashicorp/hcl/v2/hclparse"
+	"github.com/oam-dev/terraform-config-inspect/tfconfig"
 	terraformv1beta1 "github.com/oam-dev/terraform-controller/api/v1beta1"
 	kruise "github.com/openkruise/kruise-api/apps/v1alpha1"
 	certmanager "github.com/wonderflow/cert-manager-api/pkg/apis/certmanager/v1"
@@ -190,3 +192,18 @@ func ReadYamlToObject(path string, object k8sruntime.Object) error {
 	}
 	return yaml.Unmarshal(data, object)
 }
+
+// ParseTerraformVariables get variables from Terraform Configuration
+func ParseTerraformVariables(configuration string) (map[string]*tfconfig.Variable, error) {
+	p := hclparse.NewParser()
+	hclFile, diagnostic := p.ParseHCL([]byte(configuration), "")
+	if diagnostic != nil {
+		return nil, errors.New(diagnostic.Error())
+	}
+	mod := tfconfig.Module{Variables: map[string]*tfconfig.Variable{}}
+	diagnostic = tfconfig.LoadModuleFromFile(hclFile, &mod)
+	if diagnostic != nil {
+		return nil, errors.New(diagnostic.Error())
+	}
+	return mod.Variables, nil
+}

pkg/utils/common/common_test.go

@@ -239,3 +239,75 @@ func TestRealtimePrintCommandOutput(t *testing.T) {
 	assert.Contains(t, string(data), hello)
 	os.Remove(logFile)
 }
+
+func TestParseTerraformVariables(t *testing.T) {
+	configuration := `
+module "rds" {
+  source = "terraform-alicloud-modules/rds/alicloud"
+  engine = "MySQL"
+  engine_version = "8.0"
+  instance_type = "rds.mysql.c1.large"
+  instance_storage = "20"
+  instance_name = var.instance_name
+  account_name = var.account_name
+  password = var.password
+}
+
+output "DB_NAME" {
+  value = module.rds.this_db_instance_name
+}
+output "DB_USER" {
+  value = module.rds.this_db_database_account
+}
+output "DB_PORT" {
+  value = module.rds.this_db_instance_port
+}
+output "DB_HOST" {
+  value = module.rds.this_db_instance_connection_string
+}
+output "DB_PASSWORD" {
+  value = module.rds.this_db_instance_port
+}
+
+variable "instance_name" {
+  description = "RDS instance name"
+  type = string
+  default = "poc"
+}
+
+variable "account_name" {
+  description = "RDS instance user account name"
+  type = "string"
+  default = "oam"
+}
+
+variable "password" {
+  description = "RDS instance account password"
+  type = "string"
+  default = "xxx"
+}
+
+variable "intVar" {
+  type = "number"
+}
+
+variable "boolVar" {
+  type = "bool"
+}
+
+variable "listVar" {
+  type = "list"
+}
+
+variable "mapVar" {
+  type = "map"
+}`
+
+	variables, err := ParseTerraformVariables(configuration)
+	assert.NoError(t, err)
+	_, passwordExisted := variables["password"]
+	assert.True(t, passwordExisted)
+
+	_, intVarExisted := variables["password"]
+	assert.True(t, intVarExisted)
+}

references/plugins/references.go

@@ -28,8 +28,6 @@ import (
 
 	"cuelang.org/go/cue"
 	"github.com/getkin/kin-openapi/openapi3"
-	"github.com/hashicorp/hcl/v2/hclparse"
-	"github.com/oam-dev/terraform-config-inspect/tfconfig"
 	"github.com/olekukonko/tablewriter"
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
@@ -588,21 +586,6 @@ func (ref *ParseReference) parseParameters(paraValue cue.Value, paramKey string,
 	return nil
 }
 
-// parseTerraformVariables get variables from Terraform Configuration
-func (ref *ParseReference) parseTerraformVariables(configuration string) (map[string]*tfconfig.Variable, error) {
-	p := hclparse.NewParser()
-	hclFile, diagnostic := p.ParseHCL([]byte(configuration), "")
-	if diagnostic != nil {
-		return nil, errors.New(diagnostic.Error())
-	}
-	mod := tfconfig.Module{Variables: map[string]*tfconfig.Variable{}}
-	diagnostic = tfconfig.LoadModuleFromFile(hclFile, &mod)
-	if diagnostic != nil {
-		return nil, errors.New(diagnostic.Error())
-	}
-	return mod.Variables, nil
-}
-
 // getCUEPrintableDefaultValue converts the value in `interface{}` type to be printable
 func (ref *ParseReference) getCUEPrintableDefaultValue(v interface{}) string {
 	if v == nil {
@@ -729,7 +712,7 @@ func (ref *ParseReference) parseTerraformCapabilityParameters(capability types.C
 	writeConnectionSecretToRefReferenceParameter.Required = false
 	writeConnectionSecretToRefReferenceParameter.Usage = "The secret which the cloud resource connection will be written to"
 
-	variables, err := ref.parseTerraformVariables(capability.TerraformConfiguration)
+	variables, err := common.ParseTerraformVariables(capability.TerraformConfiguration)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to generate capability properties")
 	}