Fix: generate docs for Terraform ComponentDefinition (#3051)

* Fix: generate docs for Terraform ComponentDefinition

Generated kubevela.io docs for Terraform typed ComponentDefinition
with `vela def gen-doc xxx` cli.

Signed-off-by: Zheng Xi Zhou <zzxwill@gmail.com>

* Refactor code

Signed-off-by: Zheng Xi Zhou <zzxwill@gmail.com>

* add ut

Signed-off-by: Zheng Xi Zhou <zzxwill@gmail.com>

* add ut

Signed-off-by: Zheng Xi Zhou <zzxwill@gmail.com>

* refine cloud resource title

Signed-off-by: Zheng Xi Zhou <zzxwill@gmail.com>

hack/references/generate.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/oam-dev/kubevela/apis/types"
+	"github.com/oam-dev/kubevela/pkg/utils/common"
 	"github.com/oam-dev/kubevela/references/plugins"
 )
 
@@ -34,7 +36,13 @@ func main() {
 		path = plugins.KubeVelaIOTerraformPath
 	}
 
-	if err := ref.GenerateReferenceDocs(ctx, path); err != nil {
+	c, err := common.InitBaseRestConfig()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	if err := ref.GenerateReferenceDocs(ctx, c, path, types.DefaultKubeVelaNS); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}

pkg/controller/utils/capability.go

@@ -190,9 +190,16 @@ func GetOpenAPISchemaFromTerraformComponentDefinition(configuration string) ([]b
 // GetTerraformConfigurationFromRemote gets Terraform Configuration(HCL)
 func GetTerraformConfigurationFromRemote(name, remoteURL, remotePath string) (string, error) {
 	tmpPath := filepath.Join("./tmp/terraform", name)
+	// Check if the directory exists. If yes, remove it.
+	if _, err := os.Stat(tmpPath); err == nil {
+		err := os.RemoveAll(tmpPath)
+		if err != nil {
+			return "", errors.Wrap(err, "failed to remove the directory")
+		}
+	}
 	_, err := git.PlainClone(tmpPath, false, &git.CloneOptions{
 		URL:      remoteURL,
-		Progress: os.Stdout,
+		Progress: nil,
 	})
 	if err != nil {
 		return "", err

pkg/controller/utils/capability_test.go

@@ -377,6 +377,9 @@ variable "bbb" {
 }
 
 func TestGetTerraformConfigurationFromRemote(t *testing.T) {
+	// If you hit a panic on macOS as below, please fix it by referencing https://github.com/eisenxp/macos-golink-wrapper.
+	// panic: permission denied [recovered]
+	//    panic: permission denied
 	type want struct {
 		config string
 		err    error
@@ -419,16 +422,15 @@ variable "aaa" {
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
 			patch := ApplyFunc(git.PlainCloneContext, func(ctx context.Context, path string, isBare bool, o *git.CloneOptions) (*git.Repository, error) {
+				tmpPath := filepath.Join("./tmp/terraform", tc.name)
+				err := os.MkdirAll(tmpPath, os.ModePerm)
+				assert.NilError(t, err)
+				err = ioutil.WriteFile(filepath.Clean(filepath.Join(tmpPath, "main.tf")), tc.data, 0644)
+				assert.NilError(t, err)
 				return nil, nil
 			})
 			defer patch.Reset()
 
-			tmpPath := filepath.Join("./tmp/terraform", tc.name)
-			err := os.MkdirAll(tmpPath, os.ModePerm)
-			assert.NilError(t, err)
-			err = ioutil.WriteFile(filepath.Clean(filepath.Join(tmpPath, "main.tf")), tc.data, 0644)
-			assert.NilError(t, err)
-
 			conf, err := GetTerraformConfigurationFromRemote(tc.name, tc.url, tc.path)
 			if diff := cmp.Diff(tc.want.err, err, test.EquateErrors()); diff != "" {
 				t.Errorf("\n%s\nGetTerraformConfigurationFromRemote(...): -want error, +got error:\n%s", name, diff)

references/cli/def.go

@@ -44,6 +44,7 @@ import (
 	"github.com/oam-dev/kubevela/pkg/cue/model/sets"
 	pkgdef "github.com/oam-dev/kubevela/pkg/definition"
 	"github.com/oam-dev/kubevela/pkg/utils/common"
+	"github.com/oam-dev/kubevela/references/plugins"
 )
 
 const (
@@ -73,6 +74,7 @@ func DefinitionCommandGroup(c common.Args, order string) *cobra.Command {
 		NewDefinitionDelCommand(c),
 		NewDefinitionInitCommand(c),
 		NewDefinitionValidateCommand(c),
+		NewDefinitionGenDocCommand(c),
 	)
 	return cmd
 }
@@ -324,6 +326,40 @@ func NewDefinitionGetCommand(c common.Args) *cobra.Command {
 	return cmd
 }
 
+// NewDefinitionGenDocCommand create the `vela def gen-doc` command to generate documentation of definitions
+func NewDefinitionGenDocCommand(c common.Args) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "gen-doc NAME",
+		Short: "Generate documentation of definitions (Only Terraform typed definitions are supported)",
+		Long:  "Generate documentation of definitions",
+		Example: "1. Generate documentation for ComponentDefinition alibaba-vpc:\n" +
+			"> vela def gen-doc alibaba-vpc -n vela-system\n",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return fmt.Errorf("please specify definition name")
+			}
+
+			namespace, err := cmd.Flags().GetString(FlagNamespace)
+			if err != nil {
+				return errors.Wrapf(err, "failed to get `%s`", Namespace)
+			}
+
+			ref := &plugins.MarkdownReference{}
+			ctx := context.Background()
+			ref.DefinitionName = args[0]
+			path := plugins.KubeVelaIOTerraformPath
+
+			if err := ref.GenerateReferenceDocs(ctx, c, path, namespace); err != nil {
+				return errors.Wrap(err, "failed to generate reference docs")
+			}
+			cmd.Printf("Generated docs for %s in ./%s/%s.md\n", args[0], path, args[0])
+			return nil
+		},
+	}
+	cmd.Flags().StringP(Namespace, "n", "", "Specify the namespace of the definition.")
+	return cmd
+}
+
 // NewDefinitionListCommand create the `vela def list` command to list definition from k8s
 func NewDefinitionListCommand(c common.Args) *cobra.Command {
 	cmd := &cobra.Command{

references/cli/def_test.go

@@ -26,9 +26,8 @@ import (
 	"testing"
 	"time"
 
-	pkgdef "github.com/oam-dev/kubevela/pkg/definition"
-
 	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
 	"k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -36,6 +35,7 @@ import (
 
 	common3 "github.com/oam-dev/kubevela/apis/core.oam.dev/common"
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
+	pkgdef "github.com/oam-dev/kubevela/pkg/definition"
 	common2 "github.com/oam-dev/kubevela/pkg/utils/common"
 )
 
@@ -205,6 +205,7 @@ func TestNewDefinitionInitCommand(t *testing.T) {
 
 func TestNewDefinitionGetCommand(t *testing.T) {
 	c := initArgs()
+
 	// normal test
 	cmd := NewDefinitionGetCommand(c)
 	initCommand(cmd)
@@ -230,6 +231,15 @@ func TestNewDefinitionGetCommand(t *testing.T) {
 	}
 }
 
+func TestNewDefinitionGenDocCommand(t *testing.T) {
+	c := initArgs()
+	cmd := NewDefinitionGenDocCommand(c)
+	assert.NotNil(t, cmd.Execute())
+
+	cmd.SetArgs([]string{"alibaba-xxxxxxx"})
+	assert.NotNil(t, cmd.Execute())
+}
+
 func TestNewDefinitionListCommand(t *testing.T) {
 	c := initArgs()
 	// normal test

references/plugins/cluster.go

@@ -357,6 +357,9 @@ func GetCapabilityByName(ctx context.Context, c common.Args, capabilityName stri
 		}
 		return capability, nil
 	}
+	if ns == types.DefaultKubeVelaNS {
+		return nil, fmt.Errorf("could not find %s in namespace %s", capabilityName, ns)
+	}
 	return nil, fmt.Errorf("could not find %s in namespace %s, or %s", capabilityName, ns, types.DefaultKubeVelaNS)
 }
 

references/plugins/reference_test.go

@@ -487,6 +487,42 @@ func TestPrepareTerraformOutputs(t *testing.T) {
 				t.Errorf("prepareTerraformOutputs(...): -want, +got:\n%s\n", cmp.Diff(tc.expect, content))
 			}
 		})
+	}
+}
 
+func TestMakeReadableTitle(t *testing.T) {
+	type args struct {
+		title string
+	}
+	testcases := []struct {
+		args args
+		want string
+	}{
+		{
+			args: args{
+				title: "abc",
+			},
+			want: "Abc",
+		},
+		{
+			args: args{
+				title: "abc-def",
+			},
+			want: "Abc-Def",
+		},
+		{
+			args: args{
+				title: "alibaba-def-ghi",
+			},
+			want: "Alibaba Cloud DEF-GHI",
+		},
+	}
+	for _, tc := range testcases {
+		t.Run("", func(t *testing.T) {
+			title := makeReadableTitle(tc.args.title)
+			if title != tc.want {
+				t.Errorf("makeReadableTitle(...): -want, +got:\n%s\n", cmp.Diff(tc.want, title))
+			}
+		})
 	}
 }

references/plugins/references.go

@@ -365,24 +365,21 @@ func setDisplayFormat(format string) {
 }
 
 // GenerateReferenceDocs generates reference docs
-func (ref *MarkdownReference) GenerateReferenceDocs(ctx context.Context, baseRefPath string) error {
+func (ref *MarkdownReference) GenerateReferenceDocs(ctx context.Context, c common.Args, baseRefPath string, namespace string) error {
 	var (
 		caps []types.Capability
+		err  error
 	)
-	c, err := common.InitBaseRestConfig()
-	if err != nil {
-		return err
-	}
 
 	if ref.DefinitionName == "" {
 		caps, err = LoadAllInstalledCapability("default", c)
 		if err != nil {
-			return fmt.Errorf("failed to generate reference docs for all capabilities: %w", err)
+			return fmt.Errorf("failed to get all capabilityes: %w", err)
 		}
 	} else {
-		cap, err := GetCapabilityByName(ctx, c, ref.DefinitionName, types.DefaultKubeVelaNS)
+		cap, err := GetCapabilityByName(ctx, c, ref.DefinitionName, namespace)
 		if err != nil {
-			return fmt.Errorf("failed to generate reference docs for capability %s: %w", ref.DefinitionName, err)
+			return fmt.Errorf("failed to get capability capability %s: %w", ref.DefinitionName, err)
 		}
 		caps = []types.Capability{*cap}
 	}
@@ -393,28 +390,19 @@ func (ref *MarkdownReference) GenerateReferenceDocs(ctx context.Context, baseRef
 // CreateMarkdown creates markdown based on capabilities
 func (ref *MarkdownReference) CreateMarkdown(ctx context.Context, caps []types.Capability, baseRefPath, referenceSourcePath string) error {
 	setDisplayFormat("markdown")
-	var capabilityType string
 	for i, c := range caps {
-		switch c.Type {
-		case types.TypeWorkload:
-			capabilityType = WorkloadTypePath
-		case types.TypeComponentDefinition:
-			capabilityType = ComponentDefinitionTypePath
-		case types.TypeTrait:
-			capabilityType = TraitPath
-		default:
+		if c.Type != types.TypeWorkload && c.Type != types.TypeComponentDefinition && c.Type != types.TypeTrait {
 			return fmt.Errorf("the type of the capability is not right")
 		}
 
 		fileName := fmt.Sprintf("%s.md", c.Name)
-		filePath := filepath.Join(baseRefPath, capabilityType)
-		if _, err := os.Stat(filePath); err != nil && os.IsNotExist(err) {
-			if err := os.MkdirAll(filePath, 0750); err != nil {
+		if _, err := os.Stat(baseRefPath); err != nil && os.IsNotExist(err) {
+			if err := os.MkdirAll(baseRefPath, 0750); err != nil {
 				return err
 			}
 		}
 
-		markdownFile := filepath.Join(baseRefPath, capabilityType, fileName)
+		markdownFile := filepath.Join(baseRefPath, fileName)
 		f, err := os.OpenFile(filepath.Clean(markdownFile), os.O_WRONLY|os.O_CREATE, 0600)
 		if err != nil {
 			return fmt.Errorf("failed to open file %s: %w", markdownFile, err)
@@ -424,7 +412,7 @@ func (ref *MarkdownReference) CreateMarkdown(ctx context.Context, caps []types.C
 		}
 		capName := c.Name
 		refContent = ""
-		capNameInTitle := strings.Title(capName)
+		capNameInTitle := makeReadableTitle(capName)
 		switch c.Category {
 		case types.CUECategory:
 			cueValue, err := common.GetCUEParameterValue(c.CueTemplate)
@@ -483,6 +471,15 @@ func (ref *MarkdownReference) CreateMarkdown(ctx context.Context, caps []types.C
 	return nil
 }
 
+func makeReadableTitle(title string) string {
+	const alibabaCloud = "alibaba-"
+	if strings.HasPrefix(title, alibabaCloud) {
+		cloudResource := strings.Replace(title, alibabaCloud, "", 1)
+		return "Alibaba Cloud " + strings.ToUpper(cloudResource)
+	}
+	return strings.Title(title)
+}
+
 // prepareParameter prepares the table content for each property
 func (ref *MarkdownReference) prepareParameter(tableName string, parameterList []ReferenceParameter, category types.CapabilityCategory) string {
 	refContent := fmt.Sprintf("\n\n%s\n\n", tableName)