diff --git a/e2e/capability/capability_test.go b/e2e/capability/capability_test.go deleted file mode 100644 index 84d8e33a9..000000000 --- a/e2e/capability/capability_test.go +++ /dev/null @@ -1,153 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "fmt" - - "github.com/oam-dev/kubevela/apis/types" - "github.com/oam-dev/kubevela/e2e" - "github.com/oam-dev/kubevela/references/apis" - - "github.com/onsi/ginkgo" - "github.com/onsi/gomega" -) - -var ( - capabilityCenterBasic = apis.CapabilityCenterMeta{ - Name: "capability-center-e2e-basic", - URL: "https://github.com/oam-dev/kubevela/tree/master/pkg/plugins/testdata", - } - - websvcCapability = types.Capability{ - Name: "webservice.testapps", - Type: types.TypeWorkload, - } - - scaleCapability = types.Capability{ - Name: "scaler", - Type: types.TypeTrait, - } - - routeCapability = types.Capability{ - Name: "routes.test", - Type: types.TypeTrait, - } - - ingressCapability = types.Capability{ - Name: "ingress.test", - Type: types.TypeTrait, - } -) - -// TODO: change this into a mock UT to avoid remote call. - -var _ = ginkgo.Describe("Capability", func() { - ginkgo.Context("capability center", func() { - ginkgo.It("add a capability center", func() { - cli := fmt.Sprintf("vela cap center config %s %s", capabilityCenterBasic.Name, capabilityCenterBasic.URL) - output, err := e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedOutput1 := fmt.Sprintf("Successfully configured capability center %s and sync from remote", capabilityCenterBasic.Name) - gomega.Expect(output).To(gomega.ContainSubstring(expectedOutput1)) - }) - - ginkgo.It("list capability centers", func() { - cli := "vela cap center ls" - output, err := e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(output).To(gomega.ContainSubstring("NAME")) - gomega.Expect(output).To(gomega.ContainSubstring("ADDRESS")) - gomega.Expect(output).To(gomega.ContainSubstring(capabilityCenterBasic.Name)) - gomega.Expect(output).To(gomega.ContainSubstring(capabilityCenterBasic.URL)) - }) - }) - - ginkgo.Context("capability", func() { - ginkgo.It("install a workload capability to cluster", func() { - cli := fmt.Sprintf("vela cap install %s/%s", capabilityCenterBasic.Name, websvcCapability.Name) - output, err := e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedSubStr1 := fmt.Sprintf("Installing %s capability", websvcCapability.Type) - expectedSubStr2 := fmt.Sprintf("Successfully installed capability %s from %s", websvcCapability.Name, capabilityCenterBasic.Name) - gomega.Expect(output).To(gomega.ContainSubstring(expectedSubStr1)) - gomega.Expect(output).To(gomega.ContainSubstring(expectedSubStr2)) - }) - - ginkgo.It("install a trait capability to cluster", func() { - cli := fmt.Sprintf("vela cap install %s/%s", capabilityCenterBasic.Name, scaleCapability.Name) - output, err := e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedSubStr1 := fmt.Sprintf("Installing %s capability", scaleCapability.Type) - expectedSubStr2 := fmt.Sprintf("Successfully installed capability %s from %s", scaleCapability.Name, capabilityCenterBasic.Name) - gomega.Expect(output).To(gomega.ContainSubstring(expectedSubStr1)) - gomega.Expect(output).To(gomega.ContainSubstring(expectedSubStr2)) - }) - - ginkgo.It("install a trait capability without definition reference to cluster", func() { - cli := fmt.Sprintf("vela cap install %s/%s", capabilityCenterBasic.Name, ingressCapability.Name) - output, err := e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedSubStr1 := fmt.Sprintf("Installing %s capability", ingressCapability.Type) - expectedSubStr2 := fmt.Sprintf("Successfully installed capability %s from %s", ingressCapability.Name, capabilityCenterBasic.Name) - gomega.Expect(output).To(gomega.ContainSubstring(expectedSubStr1)) - gomega.Expect(output).To(gomega.ContainSubstring(expectedSubStr2)) - }) - - ginkgo.It("list all capabilities", func() { - cli := fmt.Sprintf("vela cap ls %s", capabilityCenterBasic.Name) - output, err := e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(output).To(gomega.ContainSubstring("NAME")) - gomega.Expect(output).To(gomega.ContainSubstring("CENTER")) - gomega.Expect(output).To(gomega.ContainSubstring(websvcCapability.Name)) - gomega.Expect(output).To(gomega.ContainSubstring(ingressCapability.Name)) - gomega.Expect(output).To(gomega.ContainSubstring(scaleCapability.Name)) - gomega.Expect(output).To(gomega.ContainSubstring(routeCapability.Name)) - gomega.Expect(output).To(gomega.ContainSubstring("installed")) - }) - - ginkgo.It("uninstall a workload capability from cluster", func() { - cli := fmt.Sprintf("vela cap uninstall %s", websvcCapability.Name) - output, err := e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedSubStr := fmt.Sprintf("Successfully uninstalled capability %s", websvcCapability.Name) - gomega.Expect(output).To(gomega.ContainSubstring(expectedSubStr)) - }) - - ginkgo.It("uninstall a trait capability from cluster", func() { - cli := fmt.Sprintf("vela cap uninstall %s", ingressCapability.Name) - output, err := e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedSubStr := fmt.Sprintf("Successfully uninstalled capability %s", ingressCapability.Name) - gomega.Expect(output).To(gomega.ContainSubstring(expectedSubStr)) - - // unstall other installed test capability - cli = fmt.Sprintf("vela cap uninstall %s", scaleCapability.Name) - _, err = e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - - ginkgo.It("delete a capability center", func() { - cli := fmt.Sprintf("vela cap center remove %s", capabilityCenterBasic.Name) - output, err := e2e.Exec(cli) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - expectedOutput := fmt.Sprintf("%s capability center removed successfully", capabilityCenterBasic.Name) - gomega.Expect(output).To(gomega.ContainSubstring(expectedOutput)) - }) - }) -}) diff --git a/e2e/plugin/plugin_test.go b/e2e/plugin/plugin_test.go index 4b8560508..9801d2c97 100644 --- a/e2e/plugin/plugin_test.go +++ b/e2e/plugin/plugin_test.go @@ -19,7 +19,6 @@ package plugin import ( "fmt" "os" - "os/exec" "time" . "github.com/onsi/ginkgo" @@ -172,61 +171,6 @@ var _ = Describe("Test Kubectl Plugin", func() { Expect(output).ShouldNot(ContainSubstring("mySecretKey")) }) }) - - Context("Test kubectl vela comp discover", func() { - It("Test list components in local registry", func() { - output, err := e2e.Exec("kubectl-vela comp --discover --url=" + testRegistryPath) - Expect(err).NotTo(HaveOccurred()) - Expect(output).Should(ContainSubstring("Showing components from registry")) - Expect(output).Should(ContainSubstring("fake-workload")) - }) - }) - Context("Test kubectl vela trait discover", func() { - It("Test list traits in local registry", func() { - output, err := e2e.Exec("kubectl-vela trait --discover --url=" + testRegistryPath) - Expect(err).NotTo(HaveOccurred()) - Expect(output).Should(ContainSubstring("Showing traits from registry")) - Expect(output).Should(ContainSubstring("dynamic-sa")) - }) - }) - Context("Test kubectl vela comp and trait install", func() { - It("Test install a sample component", func() { - output, err := e2e.Exec("kubectl-vela comp get cloneset --url=" + testRegistryPath) - Expect(err).NotTo(HaveOccurred()) - Expect(output).Should(ContainSubstring("Successfully install component: cloneset")) - }) - It("Test install a sample trait", func() { - output, err := e2e.Exec("kubectl-vela trait get init-container --url=" + testRegistryPath) - Expect(err).NotTo(HaveOccurred()) - Expect(output).Should(ContainSubstring("Successfully install trait: init-container")) - }) - }) - Context("Test kubectl vela list installed comp and trait", func() { - It("Test list installed component", func() { - output, err := e2e.Exec("kubectl-vela comp --url=" + testRegistryPath) - Expect(err).NotTo(HaveOccurred()) - Expect(output).Should(ContainSubstring("cloneset")) - }) - It("Test list installed trait", func() { - output, err := e2e.Exec("kubectl-vela trait --url=" + testRegistryPath) - Expect(err).NotTo(HaveOccurred()) - Expect(output).Should(ContainSubstring("init-container")) - }) - }) - Context("Test uninstall vela trait", func() { - It("Clean the sample component", func() { - cmd := exec.Command("kubectl", "delete", "componentDefinition", "cloneset", "-n", "vela-system") - output, err := cmd.Output() - Expect(err).NotTo(HaveOccurred()) - Expect(output).Should(ContainSubstring("componentdefinition.core.oam.dev \"cloneset\" deleted")) - }) - It("Clean the sample trait", func() { - cmd := exec.Command("kubectl", "delete", "traitDefinition", "init-container", "-n", "vela-system") - output, err := cmd.Output() - Expect(err).NotTo(HaveOccurred()) - Expect(output).Should(ContainSubstring("traitdefinition.core.oam.dev \"init-container\" deleted")) - }) - }) }) var application = ` diff --git a/e2e/capability/capability_suite_test.go b/e2e/registry/registry_suite_test.go similarity index 94% rename from e2e/capability/capability_suite_test.go rename to e2e/registry/registry_suite_test.go index 6c5500b54..befbf2235 100644 --- a/e2e/capability/capability_suite_test.go +++ b/e2e/registry/registry_suite_test.go @@ -25,5 +25,5 @@ import ( func TestEnv(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "Capability Suite") + ginkgo.RunSpecs(t, "Registry Suite") } diff --git a/e2e/registry/registry_test.go b/e2e/registry/registry_test.go new file mode 100644 index 000000000..e11df76fe --- /dev/null +++ b/e2e/registry/registry_test.go @@ -0,0 +1,149 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os/exec" + + "github.com/oam-dev/kubevela/e2e" + "github.com/oam-dev/kubevela/references/apis" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var ( + registryConfigs = []apis.RegistryConfig{ + { + Name: "e2e-oss-registry", + URL: "oss://registry.e2e.net", + Token: "", + }, + { + Name: "e2e-github-registry", + URL: "https://github.com/oam-dev/catalog/tree/master/traits", + Token: "", + }, + } +) + +var testTrait = "crd-manual-scaler" + +// TODO: change this into a mock UT to avoid remote call. + +var _ = Describe("test registry and trait/comp command", func() { + Context("registry", func() { + It("add and remove registry config", func() { + for _, config := range registryConfigs { + cli := fmt.Sprintf("vela registry config %s %s", config.Name, config.URL) + output, err := e2e.Exec(cli) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring(fmt.Sprintf("Successfully configured registry %s", config.Name))) + } + }) + + It("list registry config", func() { + cli := "vela registry ls" + output, err := e2e.Exec(cli) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("NAME")) + Expect(output).To(ContainSubstring("URL")) + for _, config := range registryConfigs { + Expect(output).To(ContainSubstring(config.Name)) + Expect(output).To(ContainSubstring(config.URL)) + } + }) + + It("remove registry config", func() { + for _, config := range registryConfigs { + cli := fmt.Sprintf("vela registry remove %s", config.Name) + output, err := e2e.Exec(cli) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring(fmt.Sprintf("Successfully remove registry %s", config.Name))) + } + + }) + }) + + Context("list and install trait from registry", func() { + It("list trait from cluster", func() { + cli := "vela trait" + output, err := e2e.Exec(cli) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("NAME")) + Expect(output).To(ContainSubstring("APPLIES-TO")) + Expect(output).To(ContainSubstring("pvc")) + Expect(output).To(ContainSubstring("[deployments.apps]")) + }) + It("list trait from default registry", func() { + cli := "vela trait --discover" + output, err := e2e.Exec(cli) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("Showing trait definition from registry: default")) + Expect(output).To(ContainSubstring("NAME")) + Expect(output).To(ContainSubstring("APPLIES-TO")) + Expect(output).To(ContainSubstring("STATUS")) + Expect(output).To(ContainSubstring("autoscale")) + Expect(output).To(ContainSubstring("[deployments.apps]")) + }) + + It("install traits to cluster", func() { + cli := fmt.Sprintf("vela trait get %s", testTrait) + output, err := e2e.Exec(cli) + Expect(err).NotTo(HaveOccurred()) + expectedSubStr1 := fmt.Sprintf("Installing trait %s", testTrait) + expectedSubStr2 := fmt.Sprintf("Successfully install trait: %s", testTrait) + Expect(output).To(ContainSubstring(expectedSubStr1)) + Expect(output).To(ContainSubstring(expectedSubStr2)) + }) + + It("Clean the test trait", func() { + cmd := exec.Command("kubectl", "delete", "traitDefinition", testTrait, "-n", "vela-system") + output, err := cmd.Output() + Expect(err).NotTo(HaveOccurred()) + Expect(output).Should(ContainSubstring("traitdefinition.core.oam.dev \"" + testTrait + "\" deleted")) + }) + + It("test list trait in raw url", func() { + cli := "vela trait --discover --url=oss://registry.kubevela.net" + output, err := e2e.Exec(cli) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("Showing trait definition from url"), ContainSubstring("oss://registry.kubevela.net")) + }) + + }) + + Context("test list component definition", func() { + It("test list installed component definition", func() { + cli := "vela comp" + output, err := e2e.Exec(cli) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ContainSubstring("NAME")) + Expect(output).To(ContainSubstring("DEFINITION")) + Expect(output).To(ContainSubstring("raw")) + Expect(output).To(ContainSubstring("deployments.apps")) + }) + It("test list with label", func() { + cli := "vela comp --label type=terraform" + output, err := e2e.Exec(cli) + Expect(err).NotTo(HaveOccurred()) + Expect(output).NotTo(ContainSubstring("raw")) + Expect(output).To(ContainSubstring("alibaba-ack")) + }) + }) +}) diff --git a/pkg/plugin/cli/cli.go b/pkg/plugin/cli/cli.go index 08d139d18..b8cb69285 100644 --- a/pkg/plugin/cli/cli.go +++ b/pkg/plugin/cli/cli.go @@ -60,8 +60,9 @@ func NewCommand() *cobra.Command { NewDryRunCommand(commandArgs, ioStream), NewLiveDiffCommand(commandArgs, ioStream), NewCapabilityShowCommand(commandArgs, ioStream), - NewCompCommand(commandArgs, ioStream), - NewTraitCommand(commandArgs, ioStream), + cli.NewComponentsCommand(commandArgs, ioStream), + cli.NewTraitCommand(commandArgs, ioStream), + cli.NewRegistryCommand(ioStream), NewVersionCommand(), NewHelpCommand(), ) diff --git a/pkg/plugin/cli/comp.go b/pkg/plugin/cli/comp.go deleted file mode 100644 index 6e684e125..000000000 --- a/pkg/plugin/cli/comp.go +++ /dev/null @@ -1,75 +0,0 @@ -/* - Copyright 2021. The KubeVela Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cli - -import ( - "github.com/spf13/cobra" - - "github.com/oam-dev/kubevela/apis/types" - common2 "github.com/oam-dev/kubevela/pkg/utils/common" - cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" - "github.com/oam-dev/kubevela/references/cli" -) - -// NewCompCommand creates `comp` command -func NewCompCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "comp", - DisableFlagsInUseLine: true, - Short: "Show components in capability registry", - Long: "Show components in capability registry", - Example: "kubectl vela comp", - RunE: func(cmd *cobra.Command, args []string) error { - isDiscover, _ := cmd.Flags().GetBool("discover") - url, _ := cmd.PersistentFlags().GetString("url") - err := cli.PrintComponentListFromRegistry(isDiscover, url, ioStreams) - return err - }, - Annotations: map[string]string{ - types.TagCommandType: types.TypePlugin, - }, - } - cmd.SetOut(ioStreams.Out) - cmd.AddCommand( - NewCompGetCommand(c, ioStreams), - ) - cmd.Flags().Bool("discover", false, "discover traits in registries") - cmd.PersistentFlags().String("url", cli.DefaultRegistry, "specify the registry URL") - return cmd -} - -// NewCompGetCommand creates `comp get` command -func NewCompGetCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "get ", - Short: "get component from registry", - Long: "get component from registry", - Example: "kubectl vela comp get ", - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - ioStreams.Error("you must specify a" + - " component name") - return nil - } - name := args[0] - url, _ := cmd.Flags().GetString("url") - - return cli.InstallCompByName(c, ioStreams, name, url) - }, - } - return cmd -} diff --git a/pkg/plugin/cli/trait.go b/pkg/plugin/cli/trait.go deleted file mode 100644 index 995233d16..000000000 --- a/pkg/plugin/cli/trait.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - Copyright 2021. The KubeVela Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package cli - -import ( - "github.com/spf13/cobra" - - "github.com/oam-dev/kubevela/apis/types" - common2 "github.com/oam-dev/kubevela/pkg/utils/common" - cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" - "github.com/oam-dev/kubevela/references/cli" -) - -// NewTraitCommand creates `trait` command -func NewTraitCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "trait", - DisableFlagsInUseLine: true, - Short: "Show traits in capability registry", - Long: "Show traits in capability registry", - Example: "kubectl vela trait", - RunE: func(cmd *cobra.Command, args []string) error { - isDiscover, _ := cmd.Flags().GetBool("discover") - url, _ := cmd.PersistentFlags().GetString("url") - err := cli.PrintTraitListFromRegistry(isDiscover, url, ioStreams) - return err - }, - Annotations: map[string]string{ - types.TagCommandType: types.TypePlugin, - }, - } - cmd.SetOut(ioStreams.Out) - cmd.AddCommand( - NewTraitGetCommand(c, ioStreams), - ) - cmd.Flags().Bool("discover", false, "discover traits in registries") - cmd.PersistentFlags().String("url", cli.DefaultRegistry, "specify the registry URL") - return cmd -} - -// NewTraitGetCommand creates `trait get` command -func NewTraitGetCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "get ", - Short: "get trait from registry", - Long: "get trait from registry", - Example: "kubectl vela trait get ", - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - ioStreams.Error("you must specify the trait name") - return nil - } - name := args[0] - url, _ := cmd.Flags().GetString("url") - - return cli.InstallTraitByName(c, ioStreams, name, url) - }, - } - return cmd -} diff --git a/references/apis/types.go b/references/apis/types.go index ee1e93b04..97cdc8481 100644 --- a/references/apis/types.go +++ b/references/apis/types.go @@ -96,8 +96,9 @@ type CapabilityMeta struct { CapabilityCenterName string `json:"capabilityCenterName,omitempty"` } -// CapabilityCenterMeta used for dashboard restful API server -type CapabilityCenterMeta struct { - Name string `json:"name"` - URL string `json:"url"` +// RegistryConfig is used to store registry config in file +type RegistryConfig struct { + Name string `json:"name"` + URL string `json:"url"` + Token string `json:"token"` } diff --git a/references/cli/capability.go b/references/cli/capability.go deleted file mode 100644 index c42620790..000000000 --- a/references/cli/capability.go +++ /dev/null @@ -1,292 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cli - -import ( - "errors" - "fmt" - "strings" - - "github.com/spf13/cobra" - - "github.com/oam-dev/kubevela/apis/types" - "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" - common2 "github.com/oam-dev/kubevela/pkg/utils/common" - cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" - "github.com/oam-dev/kubevela/references/common" -) - -// CapabilityCommandGroup commands for capability center -func CapabilityCommandGroup(c common2.Args, ioStream cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "cap", - Short: "Manage capability centers and installing/uninstalling capabilities", - Long: "Manage capability centers and installing/uninstalling capabilities", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return c.SetConfig() - }, - Annotations: map[string]string{ - types.TagCommandType: types.TypeCap, - }, - } - cmd.AddCommand( - NewCenterCommand(ioStream), - NewCapListCommand(c, ioStream), - NewCapInstallCommand(c, ioStream), - NewCapUninstallCommand(c, ioStream), - ) - return cmd -} - -// NewCenterCommand Manage Capability Center -func NewCenterCommand(ioStream cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "center ", - Short: "Manage Capability Center", - Long: "Manage Capability Center with config, sync, list", - } - cmd.AddCommand( - NewCapCenterConfigCommand(ioStream), - NewCapCenterSyncCommand(ioStream), - NewCapCenterListCommand(ioStream), - NewCapCenterRemoveCommand(ioStream), - ) - return cmd -} - -// NewCapCenterConfigCommand Configure (add if not exist) a capability center, default is local (built-in capabilities) -func NewCapCenterConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "config ", - Short: "Configure (add if not exist) a capability center, default is local (built-in capabilities)", - Long: "Configure (add if not exist) a capability center, default is local (built-in capabilities)", - Example: `vela cap center config mycenter https://github.com/oam-dev/catalog/tree/master/registry`, - RunE: func(cmd *cobra.Command, args []string) error { - argsLength := len(args) - if argsLength < 2 { - return errors.New("please set capability center with and ") - } - capName := args[0] - capURL := args[1] - token := cmd.Flag("token").Value.String() - if err := common.AddCapabilityCenter(capName, capURL, token); err != nil { - return err - } - ioStreams.Infof("Successfully configured capability center %s and sync from remote\n", capName) - return nil - }, - } - AddTokenVarFlags(cmd) - return cmd -} - -// NewCapInstallCommand Install capability into cluster -func NewCapInstallCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "install
/", - Short: "Install capability into cluster", - Long: "Install capability into cluster", - Example: `vela cap install mycenter/route`, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return c.SetConfig() - }, - RunE: func(cmd *cobra.Command, args []string) error { - var err error - argsLength := len(args) - if argsLength < 1 { - return errors.New("you must specify
/ for capability you want to install") - } - newClient, err := c.GetClient() - if err != nil { - return err - } - mapper, err := discoverymapper.New(c.Config) - if err != nil { - return err - } - if _, err = common.AddCapabilityIntoCluster(newClient, mapper, args[0]); err != nil { - return err - } - return nil - }, - } - AddTokenVarFlags(cmd) - return cmd -} - -// NewCapUninstallCommand Uninstall capability from cluster -func NewCapUninstallCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "uninstall ", - Short: "Uninstall capability from cluster", - Long: "Uninstall capability from cluster", - Example: `vela cap uninstall route`, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return c.SetConfig() - }, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return errors.New("you must specify for capability you want to uninstall") - } - newClient, err := c.GetClient() - if err != nil { - return err - } - name := args[0] - if strings.Contains(name, "/") { - l := strings.Split(name, "/") - if len(l) > 2 { - return fmt.Errorf("invalid format '%s', you can't contain more than one / in name", name) - } - name = l[1] - } - env, err := GetFlagEnvOrCurrent(cmd, c) - if err != nil { - return err - } - return common.RemoveCapability(env.Namespace, c, newClient, name, ioStreams) - }, - } - AddTokenVarFlags(cmd) - return cmd -} - -// NewCapCenterSyncCommand Sync capabilities from remote center, default to sync all centers -func NewCapCenterSyncCommand(ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "sync [centerName]", - Short: "Sync capabilities from remote center, default to sync all centers", - Long: "Sync capabilities from remote center, default to sync all centers", - Example: `vela cap center sync mycenter`, - RunE: func(cmd *cobra.Command, args []string) error { - var specified string - if len(args) > 0 { - specified = args[0] - } - if err := common.SyncCapabilityCenter(specified); err != nil { - return err - } - ioStreams.Info("sync finished") - return nil - }, - } - return cmd -} - -// NewCapListCommand List capabilities from cap-center -func NewCapListCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "ls [cap-center]", - Short: "List capabilities from cap-center", - Long: "List capabilities from cap-center", - Example: `vela cap ls`, - RunE: func(cmd *cobra.Command, args []string) error { - var repoName string - if len(args) > 0 { - repoName = args[0] - } - env, err := GetFlagEnvOrCurrent(cmd, c) - if err != nil { - return err - } - - err = printCenterCapabilities(env.Namespace, repoName, c, ioStreams, nil, "") - if err != nil { - return err - } - return nil - }, - } - return cmd -} - -// NewCapCenterListCommand List all capability centers -func NewCapCenterListCommand(ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "ls", - Short: "List all capability centers", - Long: "List all configured capability centers", - Example: `vela cap center ls`, - RunE: func(cmd *cobra.Command, args []string) error { - return listCapCenters(ioStreams) - }, - } - return cmd -} - -// NewCapCenterRemoveCommand Remove specified capability center -func NewCapCenterRemoveCommand(ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "remove ", - Short: "Remove specified capability center", - Long: "Remove specified capability center", - Example: "vela cap center remove mycenter", - RunE: func(cmd *cobra.Command, args []string) error { - return removeCapCenter(args, ioStreams) - }, - } - return cmd -} - -func listCapCenters(ioStreams cmdutil.IOStreams) error { - table := newUITable() - table.MaxColWidth = 80 - table.AddRow("NAME", "ADDRESS") - capabilityCenterList, err := common.ListCapabilityCenters() - if err != nil { - return err - } - for _, c := range capabilityCenterList { - table.AddRow(c.Name, c.URL) - } - ioStreams.Info(table.String()) - return nil -} - -func removeCapCenter(args []string, ioStreams cmdutil.IOStreams) error { - if len(args) < 1 { - return errors.New("you must specify for capability center you want to remove") - } - centerName := args[0] - msg, err := common.RemoveCapabilityCenter(centerName) - if err == nil { - ioStreams.Info(msg) - } - return err -} -func printCenterCapabilities(namespace, repoName string, args common2.Args, ioStreams cmdutil.IOStreams, option *types.CapType, label string) error { - capabilityList, err := common.ListCapabilities(namespace, args, repoName) - if err != nil { - return err - } - table := newUITable() - table.AddRow("NAME", "CENTER", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO") - - for _, c := range capabilityList { - if label != "" && !common.CheckLabelExistence(c.Labels, label) { - continue - } - if option == nil { - table.AddRow(c.Name, c.Center, c.Type, c.CrdName, c.Status, c.AppliesTo) - } - if option != nil && c.Type == *option { - table.AddRow(c.Name, c.Center, c.Type, c.CrdName, c.Status, c.AppliesTo) - } - } - ioStreams.Info(table.String()) - return nil -} diff --git a/references/cli/cli.go b/references/cli/cli.go index 1fa243357..840b32381 100644 --- a/references/cli/cli.go +++ b/references/cli/cli.go @@ -91,10 +91,9 @@ func NewCommand() *cobra.Command { // Workflows NewWorkflowCommand(commandArgs, ioStream), - // Capabilities - CapabilityCommandGroup(commandArgs, ioStream), + NewRegistryCommand(ioStream), NewTemplateCommand(ioStream), - NewTraitsCommand(commandArgs, ioStream), + NewTraitCommand(commandArgs, ioStream), NewComponentsCommand(commandArgs, ioStream), NewWorkloadsCommand(commandArgs, ioStream), DefinitionCommandGroup(commandArgs), diff --git a/references/cli/components.go b/references/cli/components.go index cc08d79d8..8ba5d99a9 100644 --- a/references/cli/components.go +++ b/references/cli/components.go @@ -18,9 +18,10 @@ package cli import ( "context" - "fmt" + "encoding/json" "strings" + "github.com/pkg/errors" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -30,97 +31,117 @@ import ( core "github.com/oam-dev/kubevela/apis/core.oam.dev" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" - oamutil "github.com/oam-dev/kubevela/pkg/oam/util" common2 "github.com/oam-dev/kubevela/pkg/utils/common" cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" "github.com/oam-dev/kubevela/references/common" - "github.com/oam-dev/kubevela/references/plugins" ) // NewComponentsCommand creates `components` command func NewComponentsCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { + var isDiscover bool cmd := &cobra.Command{ - Use: "components", - Aliases: []string{"comp", "component"}, - DisableFlagsInUseLine: true, - Short: "List components", - Long: "List components", - Example: `vela components`, + Use: "components", + Aliases: []string{"comp", "component"}, + Short: "List/get components", + Long: "List components & get components in registry", + Example: `vela comp`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { return c.SetConfig() }, RunE: func(cmd *cobra.Command, args []string) error { - isDiscover, _ := cmd.Flags().GetBool("discover") - env, err := GetFlagEnvOrCurrent(cmd, c) - if err != nil { - return err + // parse label filter + if label != "" { + words := strings.Split(label, "=") + if len(words) < 2 { + return errors.New("label is invalid") + } + filter = createLabelFilter(words[0], words[1]) } - label, err := cmd.Flags().GetString(types.LabelArg) - if err != nil { - return err + var registry Registry + var err error + if isDiscover { + if regURL != "" { + ioStreams.Infof("Listing component definition from url: %s\n", regURL) + registry, err = NewRegistry(context.Background(), token, "temporary-registry", regURL) + if err != nil { + return errors.Wrap(err, "creating registry err, please check registry url") + } + } else { + ioStreams.Infof("Listing component definition from registry: %s\n", regName) + registry, err = GetRegistry(regName) + if err != nil { + return errors.Wrap(err, "get registry err") + } + } + return PrintComponentListFromRegistry(registry, ioStreams, filter) } - if label != "" && len(strings.Split(label, "=")) != 2 { - return fmt.Errorf("label %s is not in the right format", label) - } - - if !isDiscover { - return printComponentList(env.Namespace, c, ioStreams, label) - } - option := types.TypeComponentDefinition - err = printCenterCapabilities(env.Namespace, "", c, ioStreams, &option, label) - if err != nil { - return err - } - - return nil + return PrintInstalledCompDef(ioStreams, filter) }, Annotations: map[string]string{ types.TagCommandType: types.TypeCap, }, } - cmd.Flags().Bool("discover", false, "discover traits in capability centers") - cmd.Flags().String(types.LabelArg, "", "a label to filter components, the format is `--label type=terraform`") + cmd.SetOut(ioStreams.Out) + cmd.AddCommand( + NewCompGetCommand(c, ioStreams), + ) + cmd.Flags().BoolVar(&isDiscover, "discover", false, "discover traits in registries") + cmd.PersistentFlags().StringVar(®URL, "url", "", "specify the registry URL") + cmd.PersistentFlags().StringVar(®Name, "registry", DefaultRegistry, "specify the registry name") + cmd.PersistentFlags().StringVar(&token, "token", "", "specify token when using --url to specify registry url") + cmd.Flags().StringVar(&label, types.LabelArg, "", "a label to filter components, the format is `--label type=terraform`") cmd.SetOut(ioStreams.Out) return cmd } -func printComponentList(userNamespace string, c common2.Args, ioStreams cmdutil.IOStreams, label string) error { - def, err := common.ListRawComponentDefinitions(userNamespace, c) - if err != nil { - return err - } - - dm, err := c.GetDiscoveryMapper() - if err != nil { - return fmt.Errorf("get discoveryMapper error %w", err) - } - - table := newUITable() - table.AddRow("NAME", "NAMESPACE", "WORKLOAD", "DESCRIPTION") - - for _, r := range def { - if label != "" && !common.CheckLabelExistence(r.Labels, label) { - continue - } - var workload string - if r.Spec.Workload.Type != "" { - workload = r.Spec.Workload.Type - } else { - definition, err := oamutil.ConvertWorkloadGVK2Definition(dm, r.Spec.Workload.Definition) - if err != nil { - return fmt.Errorf("get workload definitionReference error %w", err) +// NewCompGetCommand creates `comp get` command +func NewCompGetCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "get ", + Short: "get component from registry", + Long: "get component from registry", + Example: "vela comp get ", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + ioStreams.Error("you must specify a component name") + return nil } - workload = definition.Name - } - table.AddRow(r.Name, r.Namespace, workload, plugins.GetDescription(r.Annotations)) + name := args[0] + var registry Registry + var err error + + if regURL != "" { + ioStreams.Infof("Getting component definition from url: %s\n", regURL) + registry, err = NewRegistry(context.Background(), token, "temporary-registry", regURL) + if err != nil { + return errors.Wrap(err, "creating registry err, please check registry url") + } + } else { + ioStreams.Infof("Getting component definition from registry: %s\n", regName) + registry, err = GetRegistry(regName) + if err != nil { + return errors.Wrap(err, "get registry err") + } + } + return errors.Wrap(InstallCompByNameFromRegistry(c, ioStreams, name, registry), "install component definition err") + + }, + } + return cmd +} + +// filterFunc to filter whether to print the capability +type filterFunc func(capability types.Capability) bool + +func createLabelFilter(key, value string) filterFunc { + return func(capability types.Capability) bool { + return capability.Labels[key] == value } - ioStreams.Info(table.String()) - return nil } // PrintComponentListFromRegistry print a table which shows all components from registry -func PrintComponentListFromRegistry(isDiscover bool, url string, ioStreams cmdutil.IOStreams) error { +func PrintComponentListFromRegistry(registry Registry, ioStreams cmdutil.IOStreams, filter filterFunc) error { var scheme = runtime.NewScheme() err := core.AddToScheme(scheme) if err != nil { @@ -135,8 +156,7 @@ func PrintComponentListFromRegistry(isDiscover bool, url string, ioStreams cmdut return err } - _, _ = ioStreams.Out.Write([]byte(fmt.Sprintf("Showing components from registry: %s\n", url))) - caps, err := getCapsFromRegistry(url) + caps, err := registry.ListCaps() if err != nil { return err } @@ -147,12 +167,12 @@ func PrintComponentListFromRegistry(isDiscover bool, url string, ioStreams cmdut return err } table := newUITable() - if isDiscover { - table.AddRow("NAME", "REGISTRY", "DEFINITION") - } else { - table.AddRow("NAME", "DEFINITION") - } + table.AddRow("NAME", "REGISTRY", "DEFINITION", "STATUS") for _, c := range caps { + + if filter != nil && !filter(c) { + continue + } c.Status = uninstalled if c.Type != types.TypeComponentDefinition { continue @@ -163,26 +183,16 @@ func PrintComponentListFromRegistry(isDiscover bool, url string, ioStreams cmdut } } - if c.Status == uninstalled && isDiscover { - table.AddRow(c.Name, "default", c.CrdName) - } - if c.Status == installed && !isDiscover { - table.AddRow(c.Name, c.CrdName) - } + table.AddRow(c.Name, "default", c.CrdName, c.Status) } ioStreams.Info(table.String()) return nil } -// InstallCompByName will install given componentName comp to cluster from registry -func InstallCompByName(args common2.Args, ioStream cmdutil.IOStreams, compName, regURL string) error { - - g, err := plugins.NewRegistry(context.Background(), "", "url-registry", regURL) - if err != nil { - return err - } - capObj, data, err := g.GetCap(compName) +// InstallCompByNameFromRegistry will install given componentName comp to cluster from registry +func InstallCompByNameFromRegistry(args common2.Args, ioStream cmdutil.IOStreams, compName string, registry Registry) error { + capObj, data, err := registry.GetCap(compName) if err != nil { return err } @@ -201,3 +211,38 @@ func InstallCompByName(args common2.Args, ioStream cmdutil.IOStreams, compName, return nil } + +// PrintInstalledCompDef will print all ComponentDefinition in cluster +func PrintInstalledCompDef(io cmdutil.IOStreams, filter filterFunc) error { + var list v1beta1.ComponentDefinitionList + err := clt.List(context.Background(), &list) + if err != nil { + return errors.Wrap(err, "get component definition list error") + } + dm, err := (&common2.Args{}).GetDiscoveryMapper() + if err != nil { + return errors.Wrap(err, "get discovery mapper error") + } + + table := newUITable() + table.AddRow("NAME", "DEFINITION") + + for _, cd := range list.Items { + data, err := json.Marshal(cd) + if err != nil { + io.Infof("error encoding definition: %s\n", cd.Name) + continue + } + capa, err := ParseCapability(dm, data) + if err != nil { + io.Errorf("error parsing capability: %s\n", cd.Name) + continue + } + if filter != nil && !filter(capa) { + continue + } + table.AddRow(capa.Name, capa.CrdName) + } + io.Infof(table.String()) + return nil +} diff --git a/references/cli/help.go b/references/cli/help.go index 29b7f1983..fd532480c 100644 --- a/references/cli/help.go +++ b/references/cli/help.go @@ -62,8 +62,3 @@ func PrintHelpByTag(cmd *cobra.Command, all []*cobra.Command, tag string) { cmd.Println(table.String()) cmd.Println() } - -// AddTokenVarFlags adds token flag to a command -func AddTokenVarFlags(cmd *cobra.Command) { - cmd.PersistentFlags().StringP("token", "t", "", "Github Repo token") -} diff --git a/references/cli/registry.go b/references/cli/registry.go new file mode 100644 index 000000000..cad5b2a21 --- /dev/null +++ b/references/cli/registry.go @@ -0,0 +1,773 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software + +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +import ( + "context" + "encoding/base64" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/google/go-github/v32/github" + "golang.org/x/oauth2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/apis/types" + "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" + "github.com/oam-dev/kubevela/pkg/oam/util" + "github.com/oam-dev/kubevela/pkg/utils/common" + "github.com/oam-dev/kubevela/pkg/utils/system" + apis "github.com/oam-dev/kubevela/references/apis" + "github.com/oam-dev/kubevela/references/plugins" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + + cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" +) + +// NewRegistryCommand Manage Capability Center +func NewRegistryCommand(ioStream cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "registry ", + Short: "Manage Registry", + Long: "Manage Registry with config, remove, list", + } + cmd.AddCommand( + NewRegistryConfigCommand(ioStream), + NewRegistryListCommand(ioStream), + NewRegistryRemoveCommand(ioStream), + ) + return cmd +} + +// NewRegistryListCommand List all registry +func NewRegistryListCommand(ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "ls", + Short: "List all registry", + Long: "List all configured registry", + Example: `vela registry ls`, + RunE: func(cmd *cobra.Command, args []string) error { + return listCapRegistrys(ioStreams) + }, + } + return cmd +} + +// NewRegistryConfigCommand Configure (add if not exist) a registry, default is local (built-in capabilities) +func NewRegistryConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "config ", + Short: "Configure (add if not exist) a registry, default is local (built-in capabilities)", + Long: "Configure (add if not exist) a registry, default is local (built-in capabilities)", + Example: `vela registry config my-registry https://github.com/oam-dev/catalog/tree/master/registry`, + RunE: func(cmd *cobra.Command, args []string) error { + argsLength := len(args) + if argsLength < 2 { + return errors.New("please set registry with and ") + } + capName := args[0] + capURL := args[1] + token := cmd.Flag("token").Value.String() + if err := addRegistry(capName, capURL, token); err != nil { + return err + } + ioStreams.Infof("Successfully configured registry %s\n", capName) + return nil + }, + } + cmd.PersistentFlags().StringP("token", "t", "", "Github Repo token") + return cmd +} + +// NewRegistryRemoveCommand Remove specified registry +func NewRegistryRemoveCommand(ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Aliases: []string{"rm"}, + Use: "remove ", + Short: "Remove specified registry", + Long: "Remove specified registry", + Example: "vela registry remove mycenter", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return errors.New("you must specify for capability center you want to remove") + } + centerName := args[0] + msg, err := removeRegistry(centerName) + if err == nil { + ioStreams.Info(msg) + } + return err + }, + } + return cmd +} + +func listCapRegistrys(ioStreams cmdutil.IOStreams) error { + table := newUITable() + table.MaxColWidth = 80 + table.AddRow("NAME", "URL") + + registrys, err := ListRegistryConfig() + if err != nil { + return errors.Wrap(err, "list registry error") + } + for _, c := range registrys { + tokenShow := "" + if len(c.Token) > 0 { + tokenShow = "***" + } + table.AddRow(c.Name, c.URL, tokenShow) + } + ioStreams.Info(table.String()) + return nil +} + +// addRegistry will add a registry +func addRegistry(regName, regURL, regToken string) error { + regConfig := apis.RegistryConfig{ + Name: regName, URL: regURL, Token: regToken, + } + repos, err := ListRegistryConfig() + if err != nil { + return err + } + var updated bool + for idx, r := range repos { + if r.Name == regConfig.Name { + repos[idx] = regConfig + updated = true + break + } + } + if !updated { + repos = append(repos, regConfig) + } + if err = StoreRepos(repos); err != nil { + return err + } + return nil +} + +// removeRegistry will remove a registry from local +func removeRegistry(regName string) (string, error) { + var message string + var err error + + regConfigs, err := ListRegistryConfig() + if err != nil { + return message, err + } + found := false + for idx, r := range regConfigs { + if r.Name == regName { + regConfigs = append(regConfigs[:idx], regConfigs[idx+1:]...) + found = true + break + } + } + if !found { + return fmt.Sprintf("registry %s not found", regName), nil + } + if err = StoreRepos(regConfigs); err != nil { + return message, err + } + message = fmt.Sprintf("Successfully remove registry %s", regName) + return message, err +} + +// DefaultRegistry is default registry +const DefaultRegistry = "default" + +// Registry define a registry used to get and list types.Capability +type Registry interface { + GetName() string + GetURL() string + GetCap(addonName string) (types.Capability, []byte, error) + ListCaps() ([]types.Capability, error) +} + +// GithubRegistry is Registry's implementation treat github url as resource +type GithubRegistry struct { + URL string `json:"url"` + RegistryName string `json:"registry_name"` + client *github.Client + cfg *GithubContent + ctx context.Context +} + +// NewRegistryFromConfig return Registry interface to get capabilities +func NewRegistryFromConfig(config apis.RegistryConfig) (Registry, error) { + return NewRegistry(context.TODO(), config.Token, config.Name, config.URL) +} + +// NewRegistry will create a registry implementation +func NewRegistry(ctx context.Context, token, registryName string, regURL string) (Registry, error) { + tp, cfg, err := Parse(regURL) + if err != nil { + return nil, err + } + switch tp { + case TypeGithub: + var tc *http.Client + if token != "" { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc = oauth2.NewClient(ctx, ts) + } + return GithubRegistry{ + URL: cfg.URL, + RegistryName: registryName, + client: github.NewClient(tc), + cfg: &cfg.GithubContent, + ctx: ctx, + }, nil + case TypeOss: + var tc http.Client + return OssRegistry{ + Client: &tc, + BucketURL: fmt.Sprintf("https://%s/", cfg.BucketURL), + RegistryName: registryName, + }, nil + case TypeLocal: + _, err := os.Stat(cfg.AbsDir) + if os.IsNotExist(err) { + return LocalRegistry{}, err + } + return LocalRegistry{ + AbsPath: cfg.AbsDir, + RegistryName: registryName, + }, nil + case TypeUnknown: + return nil, fmt.Errorf("not supported url") + } + + return nil, fmt.Errorf("not supported url") +} + +// ListRegistryConfig will get all registry config stored in local +// this will return at least one config, which is DefaultRegistry +func ListRegistryConfig() ([]apis.RegistryConfig, error) { + + defaultRegistryConfig := apis.RegistryConfig{Name: DefaultRegistry, URL: "oss://registry.kubevela.net/"} + config, err := system.GetRepoConfig() + if err != nil { + return nil, err + } + data, err := os.ReadFile(filepath.Clean(config)) + if err != nil { + if os.IsNotExist(err) { + err := StoreRepos([]apis.RegistryConfig{defaultRegistryConfig}) + if err != nil { + return nil, errors.Wrap(err, "error initialize default registry") + } + return ListRegistryConfig() + } + return nil, err + } + var regConfigs []apis.RegistryConfig + if err = yaml.Unmarshal(data, ®Configs); err != nil { + return nil, err + } + haveDefault := false + for _, r := range regConfigs { + if r.URL == defaultRegistryConfig.URL { + haveDefault = true + break + } + } + if !haveDefault { + regConfigs = append(regConfigs, defaultRegistryConfig) + } + return regConfigs, nil +} + +// GetRegistry get a Registry implementation by name +func GetRegistry(regName string) (Registry, error) { + regConfigs, err := ListRegistryConfig() + if err != nil { + return nil, err + } + for _, conf := range regConfigs { + if conf.Name == regName { + return NewRegistryFromConfig(conf) + } + } + return nil, errors.Errorf("registry %s not found", regName) +} + +// GetName will return registry name +func (g GithubRegistry) GetName() string { + return g.RegistryName +} + +// GetURL will return github registry url +func (g GithubRegistry) GetURL() string { + return g.cfg.URL +} + +// ListCaps list all capabilities of registry +func (g GithubRegistry) ListCaps() ([]types.Capability, error) { + var addons []types.Capability + + itemContents, err := g.getRepoFile() + if err != nil { + return []types.Capability{}, err + } + for _, item := range itemContents { + capa, err := item.toCapability() + if err != nil { + fmt.Printf("parse definition of %s err %v\n", item.name, err) + continue + } + addons = append(addons, capa) + } + return addons, nil +} + +// GetCap return capability object and raw data specified by cap name +func (g GithubRegistry) GetCap(addonName string) (types.Capability, []byte, error) { + fileContent, _, _, err := g.client.Repositories.GetContents(context.Background(), g.cfg.Owner, g.cfg.Repo, fmt.Sprintf("%s/%s.yaml", g.cfg.Path, addonName), &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) + if err != nil { + return types.Capability{}, []byte{}, err + } + var data []byte + if *fileContent.Encoding == "base64" { + data, err = base64.StdEncoding.DecodeString(*fileContent.Content) + if err != nil { + fmt.Printf("decode github content %s err %s\n", fileContent.GetPath(), err) + } + } + repoFile := RegistryFile{ + data: data, + name: *fileContent.Name, + } + capa, err := repoFile.toCapability() + if err != nil { + return types.Capability{}, []byte{}, err + } + capa.Source = &types.Source{RepoName: g.RegistryName} + return capa, data, nil +} + +func (g *GithubRegistry) getRepoFile() ([]RegistryFile, error) { + var items []RegistryFile + _, dirs, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, g.cfg.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) + if err != nil { + return []RegistryFile{}, err + } + for _, repoItem := range dirs { + if *repoItem.Type != "file" { + continue + } + fileContent, _, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, *repoItem.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) + if err != nil { + fmt.Printf("Getting content URL %s error: %s\n", repoItem.GetURL(), err) + continue + } + var data []byte + if *fileContent.Encoding == "base64" { + data, err = base64.StdEncoding.DecodeString(*fileContent.Content) + if err != nil { + fmt.Printf("decode github content %s err %s\n", fileContent.GetPath(), err) + continue + } + } + items = append(items, RegistryFile{ + data: data, + name: *fileContent.Name, + }) + } + return items, nil +} + +// OssRegistry is Registry's implementation treat OSS url as resource +type OssRegistry struct { + *http.Client `json:"-"` + BucketURL string `json:"bucket_url"` + RegistryName string `json:"registry_name"` +} + +// GetName return name of OssRegistry +func (o OssRegistry) GetName() string { + return o.RegistryName +} + +// GetURL return URL of OssRegistry's bucket +func (o OssRegistry) GetURL() string { + return o.BucketURL +} + +// GetCap return capability object and raw data specified by cap name +func (o OssRegistry) GetCap(addonName string) (types.Capability, []byte, error) { + filename := addonName + ".yaml" + req, _ := http.NewRequestWithContext( + context.Background(), + http.MethodGet, + o.BucketURL+filename, + nil, + ) + resp, err := o.Client.Do(req) + if err != nil { + return types.Capability{}, nil, err + } + data, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + return types.Capability{}, nil, err + } + rf := RegistryFile{ + data: data, + name: filename, + } + capa, err := rf.toCapability() + if err != nil { + return types.Capability{}, nil, err + } + capa.Source = &types.Source{RepoName: o.RegistryName} + + return capa, data, nil +} + +// ListCaps list all capabilities of registry +func (o OssRegistry) ListCaps() ([]types.Capability, error) { + rfs, err := o.getRegFiles() + if err != nil { + return []types.Capability{}, errors.Wrap(err, "Get raw files fail") + } + capas := make([]types.Capability, 0) + + for _, rf := range rfs { + capa, err := rf.toCapability() + if err != nil { + fmt.Printf("[WARN] Parse file %s fail: %s\n", rf.name, err.Error()) + } + capas = append(capas, capa) + } + return capas, nil +} + +func (o OssRegistry) getRegFiles() ([]RegistryFile, error) { + req, _ := http.NewRequestWithContext( + context.Background(), + http.MethodGet, + o.BucketURL+"?list-type=2", + nil, + ) + resp, err := o.Client.Do(req) + if err != nil { + return []RegistryFile{}, err + } + data, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + return []RegistryFile{}, err + } + list := &ListBucketResult{} + err = xml.Unmarshal(data, list) + if err != nil { + return []RegistryFile{}, err + } + rfs := make([]RegistryFile, 0) + + for _, fileName := range list.File { + req, _ := http.NewRequestWithContext( + context.Background(), + http.MethodGet, + o.BucketURL+fileName, + nil, + ) + resp, err := o.Client.Do(req) + if err != nil { + fmt.Printf("[WARN] %s download fail\n", fileName) + continue + } + data, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + rf := RegistryFile{ + data: data, + name: fileName, + } + rfs = append(rfs, rf) + + } + return rfs, nil +} + +// LocalRegistry is Registry's implementation treat local url as resource +type LocalRegistry struct { + AbsPath string `json:"abs_path"` + RegistryName string `json:"registry_name"` +} + +// GetName return name of LocalRegistry +func (l LocalRegistry) GetName() string { + return l.RegistryName +} + +// GetURL return path of LocalRegistry +func (l LocalRegistry) GetURL() string { + return l.AbsPath +} + +// GetCap return capability object and raw data specified by cap name +func (l LocalRegistry) GetCap(addonName string) (types.Capability, []byte, error) { + fileName := addonName + ".yaml" + filePath := fmt.Sprintf("%s/%s", l.AbsPath, fileName) + data, err := os.ReadFile(filePath) + if err != nil { + return types.Capability{}, []byte{}, err + } + file := RegistryFile{ + data: data, + name: fileName, + } + capa, err := file.toCapability() + if err != nil { + return types.Capability{}, []byte{}, err + } + capa.Source = &types.Source{RepoName: l.RegistryName} + + return capa, data, nil +} + +// ListCaps list all capabilities of registry +func (l LocalRegistry) ListCaps() ([]types.Capability, error) { + glob := filepath.Join(filepath.Clean(l.AbsPath), "*") + files, _ := filepath.Glob(glob) + capas := make([]types.Capability, 0) + for _, file := range files { + // nolint:gosec + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + capa, err := RegistryFile{ + data: data, + name: path.Base(file), + }.toCapability() + if err != nil { + fmt.Printf("parsing file: %s err: %s\n", file, err) + continue + } + capas = append(capas, capa) + } + return capas, nil +} + +func (item RegistryFile) toCapability() (types.Capability, error) { + dm, err := (&common.Args{}).GetDiscoveryMapper() + if err != nil { + return types.Capability{}, err + } + capability, err := ParseCapability(dm, item.data) + if err != nil { + return types.Capability{}, err + } + return capability, nil +} + +// RegistryFile describes a file item in registry +type RegistryFile struct { + data []byte // file content + name string // file's name +} + +// ListBucketResult describe a file list from OSS +type ListBucketResult struct { + File []string `xml:"Contents>Key"` + Count int `xml:"KeyCount"` +} + +// Content contains different type of content needed when building Registry +type Content struct { + OssContent + GithubContent + LocalContent +} + +// LocalContent for local registry +type LocalContent struct { + AbsDir string `json:"abs_dir"` +} + +// OssContent for oss registry +type OssContent struct { + BucketURL string `json:"bucket_url"` +} + +// GithubContent for registry +type GithubContent struct { + URL string `json:"url"` + Owner string `json:"owner"` + Repo string `json:"repo"` + Path string `json:"path"` + Ref string `json:"ref"` +} + +// TypeLocal represents github +const TypeLocal = "local" + +// TypeOss represent oss +const TypeOss = "oss" + +// TypeGithub represents github +const TypeGithub = "github" + +// TypeUnknown represents parse failed +const TypeUnknown = "unknown" + +// Parse will parse config from address +func Parse(addr string) (string, *Content, error) { + URL, err := url.Parse(addr) + if err != nil { + return "", nil, err + } + l := strings.Split(strings.TrimPrefix(URL.Path, "/"), "/") + switch URL.Scheme { + case "http", "https": + switch URL.Host { + case "github.com": + // We support two valid format: + // 1. https://github.com///tree// + // 2. https://github.com/// + if len(l) < 3 { + return "", nil, errors.New("invalid format " + addr) + } + if l[2] == "tree" { + // https://github.com///tree// + if len(l) < 5 { + return "", nil, errors.New("invalid format " + addr) + } + return TypeGithub, &Content{ + GithubContent: GithubContent{ + URL: addr, + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[4:], "/"), + Ref: l[3], + }, + }, nil + } + // https://github.com/// + return TypeGithub, &Content{ + GithubContent: GithubContent{ + URL: addr, + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[2:], "/"), + Ref: "", // use default branch + }, + }, + nil + case "api.github.com": + if len(l) != 5 { + return "", nil, errors.New("invalid format " + addr) + } + //https://api.github.com/repos///contents/ + return TypeGithub, &Content{ + GithubContent: GithubContent{ + URL: addr, + Owner: l[1], + Repo: l[2], + Path: l[4], + Ref: URL.Query().Get("ref"), + }, + }, + nil + default: + } + case "oss": + return TypeOss, &Content{ + OssContent: OssContent{ + BucketURL: URL.Host, + }, + }, nil + case "file": + return TypeLocal, &Content{ + LocalContent: LocalContent{ + AbsDir: URL.Path, + }, + }, nil + + } + + return TypeUnknown, nil, nil +} + +// StoreRepos will store registry repo locally +func StoreRepos(registries []apis.RegistryConfig) error { + config, err := system.GetRepoConfig() + if err != nil { + return err + } + data, err := yaml.Marshal(registries) + if err != nil { + return err + } + //nolint:gosec + return os.WriteFile(config, data, 0644) +} + +// ParseCapability will convert config from remote center to capability +func ParseCapability(mapper discoverymapper.DiscoveryMapper, data []byte) (types.Capability, error) { + var obj = unstructured.Unstructured{Object: make(map[string]interface{})} + err := yaml.Unmarshal(data, &obj.Object) + if err != nil { + return types.Capability{}, err + } + switch obj.GetKind() { + case "ComponentDefinition": + var cd v1beta1.ComponentDefinition + err = yaml.Unmarshal(data, &cd) + if err != nil { + return types.Capability{}, err + } + var workloadDefinitionRef string + if cd.Spec.Workload.Type != "" { + workloadDefinitionRef = cd.Spec.Workload.Type + } else { + ref, err := util.ConvertWorkloadGVK2Definition(mapper, cd.Spec.Workload.Definition) + if err != nil { + return types.Capability{}, err + } + workloadDefinitionRef = ref.Name + } + return plugins.HandleDefinition(cd.Name, workloadDefinitionRef, cd.Annotations, cd.Labels, cd.Spec.Extension, types.TypeComponentDefinition, nil, cd.Spec.Schematic) + case "TraitDefinition": + var td v1beta1.TraitDefinition + err = yaml.Unmarshal(data, &td) + if err != nil { + return types.Capability{}, err + } + return plugins.HandleDefinition(td.Name, td.Spec.Reference.Name, td.Annotations, td.Labels, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads, td.Spec.Schematic) + case "ScopeDefinition": + // TODO(wonderflow): support scope definition here. + } + return types.Capability{}, fmt.Errorf("unknown definition Type %s", obj.GetKind()) +} diff --git a/references/plugins/registry_test.go b/references/cli/registry_test.go similarity index 56% rename from references/plugins/registry_test.go rename to references/cli/registry_test.go index 4a4990179..4b908875f 100644 --- a/references/plugins/registry_test.go +++ b/references/cli/registry_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package plugins +package cli import ( "context" @@ -63,3 +63,50 @@ func TestRegistry(t *testing.T) { assert.NotNil(t, data, testAddon) } } + +func TestParseURL(t *testing.T) { + cases := map[string]struct { + url string + exp *GithubContent + expType string + }{ + "api-github": { + url: "https://api.github.com/repos/oam-dev/catalog/contents/traits?ref=master", + expType: TypeGithub, + exp: &GithubContent{ + URL: "https://api.github.com/repos/oam-dev/catalog/contents/traits?ref=master", + Owner: "oam-dev", + Repo: "catalog", + Path: "traits", + Ref: "master", + }, + }, + "github-copy-path": { + url: "https://github.com/oam-dev/catalog/tree/master/repository", + expType: TypeGithub, + exp: &GithubContent{ + URL: "https://github.com/oam-dev/catalog/tree/master/repository", + Owner: "oam-dev", + Repo: "catalog", + Path: "repository", + Ref: "master", + }, + }, + "github-manual-write-path": { + url: "https://github.com/oam-dev/catalog/traits", + expType: TypeGithub, + exp: &GithubContent{ + URL: "https://github.com/oam-dev/catalog/traits", + Owner: "oam-dev", + Repo: "catalog", + Path: "traits", + }, + }, + } + for caseName, c := range cases { + tp, content, err := Parse(c.url) + assert.NoError(t, err, caseName) + assert.Equal(t, c.exp, &content.GithubContent, caseName) + assert.Equal(t, c.expType, tp, caseName) + } +} diff --git a/references/cli/traits.go b/references/cli/traits.go index 8537667da..edba23773 100644 --- a/references/cli/traits.go +++ b/references/cli/traits.go @@ -18,9 +18,10 @@ package cli import ( "context" - "fmt" + "encoding/json" "strings" + "github.com/pkg/errors" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -33,77 +34,113 @@ import ( common2 "github.com/oam-dev/kubevela/pkg/utils/common" cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" "github.com/oam-dev/kubevela/references/common" - "github.com/oam-dev/kubevela/references/plugins" ) -// NewTraitsCommand creates `traits` command -func NewTraitsCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { +var ( + regName string + regURL string + token string + label string + filter filterFunc +) + +// NewTraitCommand creates `traits` command +func NewTraitCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { + var isDiscover bool cmd := &cobra.Command{ - Use: "traits", - Aliases: []string{"trait"}, - DisableFlagsInUseLine: true, - Short: "List traits", - Long: "List traits", - Example: `vela traits`, + Use: "trait", + Aliases: []string{"traits"}, + Short: "List/get traits", + Long: "List traits & get trait in registry", + Example: `vela trait`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { return c.SetConfig() }, RunE: func(cmd *cobra.Command, args []string) error { - isDiscover, _ := cmd.Flags().GetBool("discover") - env, err := GetFlagEnvOrCurrent(cmd, c) - if err != nil { - return err - } - label, err := cmd.Flags().GetString(types.LabelArg) - if err != nil { - return err - } - if label != "" && len(strings.Split(label, "=")) != 2 { - return fmt.Errorf("label %s is not in the right format", label) - - } - if !isDiscover { - return printTraitList(env.Namespace, c, ioStreams, label) - } - option := types.TypeTrait - err = printCenterCapabilities(env.Namespace, "", c, ioStreams, &option, label) - if err != nil { - return err + // parse label filter + if label != "" { + words := strings.Split(label, "=") + if len(words) < 2 { + return errors.New("label is invalid") + } + filter = createLabelFilter(words[0], words[1]) } - return nil + var registry Registry + var err error + if isDiscover { + if regURL != "" { + ioStreams.Infof("Showing trait definition from url: %s\n", regURL) + registry, err = NewRegistry(context.Background(), token, "temporary-registry", regURL) + if err != nil { + return errors.Wrap(err, "creating registry err, please check registry url") + } + } else { + ioStreams.Infof("Showing trait definition from registry: %s\n", regName) + registry, err = GetRegistry(regName) + if err != nil { + return errors.Wrap(err, "get registry err") + } + } + return PrintTraitListFromRegistry(registry, ioStreams, filter) + + } + return PrintInstalledTraitDef(ioStreams, filter) }, Annotations: map[string]string{ types.TagCommandType: types.TypeCap, }, } - cmd.Flags().Bool("discover", false, "discover traits in capability centers") - cmd.Flags().String(types.LabelArg, "", "a label to filter components, the format is `--label type=terraform`") + cmd.SetOut(ioStreams.Out) + cmd.AddCommand( + NewTraitGetCommand(c, ioStreams), + ) + cmd.Flags().BoolVar(&isDiscover, "discover", false, "discover traits in registries") + cmd.PersistentFlags().StringVar(®URL, "url", "", "specify the registry URL") + cmd.PersistentFlags().StringVar(&token, "token", "", "specify token when using --url to specify registry url") + cmd.PersistentFlags().StringVar(®Name, "registry", DefaultRegistry, "specify the registry name") + cmd.Flags().StringVar(&label, types.LabelArg, "", "a label to filter components, the format is `--label type=terraform`") cmd.SetOut(ioStreams.Out) return cmd } -func printTraitList(userNamespace string, c common2.Args, ioStreams cmdutil.IOStreams, label string) error { - table := newUITable() - table.Wrap = true +// NewTraitGetCommand creates `trait get` command +func NewTraitGetCommand(c common2.Args, ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "get ", + Short: "get trait from registry", + Long: "get trait from registry", + Example: "vela trait get ", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + ioStreams.Error("you must specify the trait name") + return nil + } + name := args[0] + var registry Registry + var err error - traitDefinitionList, err := common.ListRawTraitDefinitions(userNamespace, c) - if err != nil { - return err + if regURL != "" { + ioStreams.Infof("Getting trait definition from url: %s\n", regURL) + registry, err = NewRegistry(context.Background(), token, "temporary-registry", regURL) + if err != nil { + return errors.Wrap(err, "creating registry err, please check registry url") + } + } else { + ioStreams.Infof("Getting trait definition from registry: %s\n", regName) + registry, err = GetRegistry(regName) + if err != nil { + return errors.Wrap(err, "get registry err") + } + } + return errors.Wrap(InstallTraitByNameFromRegistry(c, ioStreams, name, registry), "install trait definition err") + }, } - table.AddRow("NAME", "NAMESPACE", "APPLIES-TO", "CONFLICTS-WITH", "POD-DISRUPTIVE", "DESCRIPTION") - for _, t := range traitDefinitionList { - if label != "" && !common.CheckLabelExistence(t.Labels, label) { - continue - } - table.AddRow(t.Name, t.Namespace, strings.Join(t.Spec.AppliesToWorkloads, ","), strings.Join(t.Spec.ConflictsWith, ","), t.Spec.PodDisruptive, plugins.GetDescription(t.Annotations)) - } - ioStreams.Info(table.String()) - return nil + return cmd } // PrintTraitListFromRegistry print a table which shows all traits from registry -func PrintTraitListFromRegistry(isDiscover bool, url string, ioStreams cmdutil.IOStreams) error { +func PrintTraitListFromRegistry(registry Registry, ioStreams cmdutil.IOStreams, filter filterFunc) error { var scheme = runtime.NewScheme() err := core.AddToScheme(scheme) if err != nil { @@ -118,8 +155,7 @@ func PrintTraitListFromRegistry(isDiscover bool, url string, ioStreams cmdutil.I return err } - _, _ = ioStreams.Out.Write([]byte(fmt.Sprintf("Showing traits from registry: %s\n", url))) - caps, err := getCapsFromRegistry(url) + caps, err := registry.ListCaps() if err != nil { return err } @@ -131,12 +167,12 @@ func PrintTraitListFromRegistry(isDiscover bool, url string, ioStreams cmdutil.I if err != nil { return err } - if isDiscover { - table.AddRow("NAME", "REGISTRY", "DEFINITION", "APPLIES-TO") - } else { - table.AddRow("NAME", "DEFINITION", "APPLIES-TO") - } + + table.AddRow("NAME", "REGISTRY", "DEFINITION", "APPLIES-TO", "STATUS") for _, c := range caps { + if filter != nil && !filter(c) { + continue + } if c.Type != types.TypeTrait { continue } @@ -146,39 +182,16 @@ func PrintTraitListFromRegistry(isDiscover bool, url string, ioStreams cmdutil.I c.Status = installed } } - if c.Status == uninstalled && isDiscover { - table.AddRow(c.Name, "default", c.CrdName, c.AppliesTo) - } - if c.Status == installed && !isDiscover { - table.AddRow(c.Name, c.CrdName, c.AppliesTo) - } + table.AddRow(c.Name, "default", c.CrdName, c.AppliesTo, c.Status) } ioStreams.Info(table.String()) return nil } -// getCapsFromRegistry will retrieve caps from registry -func getCapsFromRegistry(regURL string) ([]types.Capability, error) { - g, err := plugins.NewRegistry(context.Background(), "", "url-registry", regURL) - if err != nil { - return []types.Capability{}, err - } - caps, err := g.ListCaps() - if err != nil { - return []types.Capability{}, err - } - return caps, nil -} - -// InstallTraitByName will install given traitName trait to cluster -func InstallTraitByName(args common2.Args, ioStream cmdutil.IOStreams, traitName, regURL string) error { - - g, err := plugins.NewRegistry(context.Background(), "", "url-registry", regURL) - if err != nil { - return err - } - capObj, data, err := g.GetCap(traitName) +// InstallTraitByNameFromRegistry will install given traitName trait to cluster +func InstallTraitByNameFromRegistry(args common2.Args, ioStream cmdutil.IOStreams, traitName string, registry Registry) error { + capObj, data, err := registry.GetCap(traitName) if err != nil { return err } @@ -200,8 +213,40 @@ func InstallTraitByName(args common2.Args, ioStream cmdutil.IOStreams, traitName return nil } -// DefaultRegistry is default capability center of kubectl-vela -var DefaultRegistry = "oss://registry.kubevela.net" +// PrintInstalledTraitDef will print all TraitDefinition in cluster +func PrintInstalledTraitDef(io cmdutil.IOStreams, filter filterFunc) error { + var list v1beta1.TraitDefinitionList + err := clt.List(context.Background(), &list) + if err != nil { + return errors.Wrap(err, "get trait definition list error") + } + dm, err := (&common2.Args{}).GetDiscoveryMapper() + if err != nil { + return errors.Wrap(err, "get discovery mapper error") + } + + table := newUITable() + table.AddRow("NAME", "APPLIES-TO") + + for _, td := range list.Items { + data, err := json.Marshal(td) + if err != nil { + io.Infof("error encoding definition: %s\n", td.Name) + continue + } + capa, err := ParseCapability(dm, data) + if err != nil { + io.Errorf("error parsing capability: %s\n", td.Name) + continue + } + if filter != nil && !filter(capa) { + continue + } + table.AddRow(capa.Name, capa.AppliesTo) + } + io.Infof(table.String()) + return nil +} const installed = "installed" const uninstalled = "uninstalled" diff --git a/references/cli/traits_test.go b/references/cli/traits_test.go index 503afbf2d..3e1f2a860 100644 --- a/references/cli/traits_test.go +++ b/references/cli/traits_test.go @@ -33,7 +33,7 @@ import ( func TestNewTraitsCommandPersistentPreRunE(t *testing.T) { io := cmdutil.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} fakeC := common2.Args{} - cmd := NewTraitsCommand(fakeC, io) + cmd := NewTraitCommand(fakeC, io) assert.Nil(t, cmd.PersistentPreRunE(new(cobra.Command), []string{})) } diff --git a/references/common/application.go b/references/common/application.go index 20d9bd32c..3d1498bae 100644 --- a/references/common/application.go +++ b/references/common/application.go @@ -23,13 +23,9 @@ import ( "fmt" "os" "path/filepath" - "sort" "strings" - "time" - "github.com/AlecAivazis/survey/v2" "github.com/pkg/errors" - "github.com/spf13/cobra" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/serializer/json" apitypes "k8s.io/apimachinery/pkg/types" @@ -40,27 +36,14 @@ import ( corev1beta1 "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/oam" - oamutil "github.com/oam-dev/kubevela/pkg/oam/util" "github.com/oam-dev/kubevela/pkg/utils/apply" "github.com/oam-dev/kubevela/pkg/utils/common" cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" - "github.com/oam-dev/kubevela/references/apis" "github.com/oam-dev/kubevela/references/appfile" "github.com/oam-dev/kubevela/references/appfile/api" "github.com/oam-dev/kubevela/references/appfile/template" ) -// nolint:golint -const ( - DefaultChosenAllSvc = "ALL SERVICES" - FlagNotSet = "FlagNotSet" - FlagIsInvalid = "FlagIsInvalid" - FlagIsValid = "FlagIsValid" -) - -type componentMetaList []apis.ComponentMeta -type applicationMetaList []apis.ApplicationMeta - // AppfileOptions is some configuration that modify options for an Appfile type AppfileOptions struct { Kubecli client.Client @@ -75,26 +58,6 @@ type BuildResult struct { scopes []oam.Object } -func (comps componentMetaList) Len() int { - return len(comps) -} -func (comps componentMetaList) Swap(i, j int) { - comps[i], comps[j] = comps[j], comps[i] -} -func (comps componentMetaList) Less(i, j int) bool { - return comps[i].CreatedTime > comps[j].CreatedTime -} - -func (a applicationMetaList) Len() int { - return len(a) -} -func (a applicationMetaList) Swap(i, j int) { - a[i], a[j] = a[j], a[i] -} -func (a applicationMetaList) Less(i, j int) bool { - return a[i].CreatedTime > a[j].CreatedTime -} - // Option is option work with dashboard api server type Option struct { // Optional filter, if specified, only components in such app will be listed @@ -112,101 +75,6 @@ type DeleteOptions struct { C common.Args } -// ListApplications lists all applications -func ListApplications(ctx context.Context, c client.Reader, opt Option) ([]apis.ApplicationMeta, error) { - var applicationMetaList applicationMetaList - var appList corev1beta1.ApplicationList - if opt.AppName != "" { - var app corev1beta1.Application - if err := c.Get(ctx, client.ObjectKey{Name: opt.AppName, Namespace: opt.Namespace}, &app); err != nil { - return applicationMetaList, err - } - appList.Items = append(appList.Items, app) - } else { - err := c.List(ctx, &appList, &client.ListOptions{Namespace: opt.Namespace}) - if err != nil { - return applicationMetaList, err - } - } - for _, a := range appList.Items { - // ignore the deleted resource - if a.GetDeletionGracePeriodSeconds() != nil { - continue - } - applicationMeta, err := RetrieveApplicationStatusByName(ctx, c, a.Name, a.Namespace) - if err != nil { - return applicationMetaList, err - } - applicationMeta.Components = nil - applicationMetaList = append(applicationMetaList, applicationMeta) - } - sort.Stable(applicationMetaList) - return applicationMetaList, nil -} - -// ListApplicationConfigurations lists all OAM ApplicationConfiguration -func ListApplicationConfigurations(ctx context.Context, c client.Reader, opt Option) (corev1alpha2.ApplicationConfigurationList, error) { - var appConfigList corev1alpha2.ApplicationConfigurationList - - if opt.AppName != "" { - var appConfig corev1alpha2.ApplicationConfiguration - if err := c.Get(ctx, client.ObjectKey{Name: opt.AppName, Namespace: opt.Namespace}, &appConfig); err != nil { - return appConfigList, err - } - appConfigList.Items = append(appConfigList.Items, appConfig) - } else { - err := c.List(ctx, &appConfigList, &client.ListOptions{Namespace: opt.Namespace}) - if err != nil { - return appConfigList, err - } - } - return appConfigList, nil -} - -// ListComponents will list all components for dashboard -func ListComponents(ctx context.Context, c client.Reader, opt Option) ([]apis.ComponentMeta, error) { - var componentMetaList componentMetaList - var appConfigList corev1alpha2.ApplicationConfigurationList - var err error - if appConfigList, err = ListApplicationConfigurations(ctx, c, opt); err != nil { - return nil, err - } - - for _, a := range appConfigList.Items { - for _, com := range a.Spec.Components { - component, _, err := oamutil.GetComponent(ctx, c, com, opt.Namespace) - if err != nil { - return componentMetaList, err - } - componentMetaList = append(componentMetaList, apis.ComponentMeta{ - Name: com.ComponentName, - Status: types.StatusDeployed, - CreatedTime: a.ObjectMeta.CreationTimestamp.String(), - Component: *component, - AppConfig: a, - App: a.Name, - }) - } - } - sort.Stable(componentMetaList) - return componentMetaList, nil -} - -// RetrieveApplicationStatusByName will get app status -func RetrieveApplicationStatusByName(ctx context.Context, c client.Reader, applicationName string, - namespace string) (apis.ApplicationMeta, error) { - var applicationMeta apis.ApplicationMeta - var app corev1beta1.Application - if err := c.Get(ctx, client.ObjectKey{Name: applicationName, Namespace: namespace}, &app); err != nil { - return applicationMeta, err - } - applicationMeta.Name = app.Name - applicationMeta.Status = string(app.Status.Phase) - applicationMeta.CreatedTime = app.CreationTimestamp.Format(time.RFC3339) - - return applicationMeta, nil -} - // DeleteApp will delete app including server side func (o *DeleteOptions) DeleteApp() (string, error) { ctx := context.Background() @@ -274,64 +142,6 @@ func (o *DeleteOptions) DeleteComponent(io cmdutil.IOStreams) (string, error) { return fmt.Sprintf("component \"%s\" deleted from \"%s\"", o.CompName, o.AppName), nil } -func chooseSvc(services []string) (string, error) { - var svcName string - services = append(services, DefaultChosenAllSvc) - prompt := &survey.Select{ - Message: "Please choose one service: ", - Options: services, - Default: DefaultChosenAllSvc, - } - err := survey.AskOne(prompt, &svcName) - if err != nil { - return "", fmt.Errorf("failed to retrieve services of the application, err %w", err) - } - return svcName, nil -} - -// GetServicesWhenDescribingApplication gets the target services list either from cli `--svc` flag or from survey -func GetServicesWhenDescribingApplication(cmd *cobra.Command, app *api.Application) ([]string, error) { - var svcFlag string - var svcFlagStatus string - // to store the value of flag `--svc` set in Cli, or selected value in survey - var targetServices []string - if svcFlag = cmd.Flag("svc").Value.String(); svcFlag == "" { - svcFlagStatus = FlagNotSet - } else { - svcFlagStatus = FlagIsInvalid - } - // all services name of the application `appName` - var services []string - for svcName := range app.Services { - services = append(services, svcName) - if svcFlag == svcName { - svcFlagStatus = FlagIsValid - targetServices = append(targetServices, svcName) - } - } - totalServices := len(services) - if svcFlagStatus == FlagNotSet && totalServices == 1 { - targetServices = services - } - if svcFlagStatus == FlagIsInvalid || (svcFlagStatus == FlagNotSet && totalServices > 1) { - if svcFlagStatus == FlagIsInvalid { - cmd.Printf("The service name '%s' is not valid\n", svcFlag) - } - chosenSvc, err := chooseSvc(services) - if err != nil { - return []string{}, err - } - - if chosenSvc == DefaultChosenAllSvc { - targetServices = services - } else { - targetServices = targetServices[:0] - targetServices = append(targetServices, chosenSvc) - } - } - return targetServices, nil -} - func saveAndLoadRemoteAppfile(url string) (*api.AppFile, error) { body, err := common.HTTPGet(context.Background(), url) if err != nil { diff --git a/references/common/capability.go b/references/common/capability.go deleted file mode 100644 index 622e29413..000000000 --- a/references/common/capability.go +++ /dev/null @@ -1,520 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package common - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/oam-dev/kubevela/pkg/utils/common" - - corev1 "k8s.io/api/core/v1" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/yaml" - - "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" - "github.com/oam-dev/kubevela/apis/types" - "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" - "github.com/oam-dev/kubevela/pkg/oam/util" - "github.com/oam-dev/kubevela/pkg/utils/helm" - "github.com/oam-dev/kubevela/pkg/utils/system" - cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" - "github.com/oam-dev/kubevela/references/apis" - "github.com/oam-dev/kubevela/references/plugins" -) - -// AddCapabilityCenter will add a cap center -func AddCapabilityCenter(capName, capURL, capToken string) error { - repos, err := plugins.LoadRepos() - if err != nil { - return err - } - config := &plugins.CapCenterConfig{ - Name: capName, - Address: capURL, - Token: capToken, - } - var updated bool - for idx, r := range repos { - if r.Name == config.Name { - repos[idx] = *config - updated = true - break - } - } - if !updated { - repos = append(repos, *config) - } - if err = plugins.StoreRepos(repos); err != nil { - return err - } - return SyncCapabilityFromCenter(capName, capURL, capToken) -} - -// SyncCapabilityFromCenter will sync all capabilities from center -func SyncCapabilityFromCenter(capName, capURL, capToken string) error { - client, err := plugins.NewCenterClient(context.Background(), capName, capURL, capToken) - if err != nil { - return err - } - return client.SyncCapabilityFromCenter() -} - -// AddCapabilityIntoCluster will add a capability into K8s cluster, it is equal to apply a definition yaml and run `vela workloads/traits` -func AddCapabilityIntoCluster(c client.Client, mapper discoverymapper.DiscoveryMapper, capability string) (string, error) { - ss := strings.Split(capability, "/") - if len(ss) < 2 { - return "", errors.New("invalid format for " + capability + ", please follow format
/") - } - repoName := ss[0] - name := ss[1] - ioStreams := cmdutil.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} - if err := InstallCapability(c, mapper, repoName, name, ioStreams); err != nil { - return "", err - } - return fmt.Sprintf("Successfully installed capability %s from %s", name, repoName), nil -} - -// InstallCapability will add a cap into K8s cluster and install it's controller(helm charts) -func InstallCapability(client client.Client, mapper discoverymapper.DiscoveryMapper, centerName, capabilityName string, ioStreams cmdutil.IOStreams) error { - dir, _ := system.GetCapCenterDir() - repoDir := filepath.Join(dir, centerName) - tp, err := GetCapabilityFromCenter(mapper, centerName, capabilityName) - if err != nil { - return err - } - defDir, _ := system.GetCapabilityDir() - fileContent, err := os.ReadFile(filepath.Clean(filepath.Join(repoDir, tp.Name+".yaml"))) - if err != nil { - return err - } - switch tp.Type { - case types.TypeComponentDefinition: - err = InstallComponentDefinition(client, fileContent, ioStreams, &tp) - if err != nil { - return err - } - case types.TypeTrait: - err = InstallTraitDefinition(client, mapper, fileContent, ioStreams, &tp) - if err != nil { - return err - } - case types.TypeScope: - // TODO(wonderflow): support install scope here - case types.TypeWorkload: - return fmt.Errorf("unsupported capability type %v", types.TypeWorkload) - default: - return fmt.Errorf("unsupported type: %v", tp.Type) - } - - success := plugins.SinkTemp2Local([]types.Capability{tp}, defDir) - if success == 1 { - ioStreams.Infof("Successfully installed capability %s from %s\n", capabilityName, centerName) - } - return nil -} - -// InstallComponentDefinition will add a component into K8s cluster and install it's controller -func InstallComponentDefinition(client client.Client, workloadData []byte, ioStreams cmdutil.IOStreams, tp *types.Capability) error { - var cd v1beta1.ComponentDefinition - var err error - if err = yaml.Unmarshal(workloadData, &cd); err != nil { - return err - } - cd.Namespace = types.DefaultKubeVelaNS - ioStreams.Info("Installing component capability " + cd.Name) - if tp.Install != nil { - tp.Source.ChartName = tp.Install.Helm.Name - if err = helm.InstallHelmChart(ioStreams, tp.Install.Helm); err != nil { - return err - } - err = addSourceIntoExtension(cd.Spec.Extension, tp.Source) - if err != nil { - return err - } - } - if cd.Spec.Workload.Type == "" { - tp.CrdInfo = &types.CRDInfo{ - APIVersion: cd.Spec.Workload.Definition.APIVersion, - Kind: cd.Spec.Workload.Definition.Kind, - } - } - if err = client.Create(context.Background(), &cd); err != nil && !apierrors.IsAlreadyExists(err) { - return err - } - return nil -} - -// InstallTraitDefinition will add a trait into K8s cluster and install it's controller -func InstallTraitDefinition(client client.Client, mapper discoverymapper.DiscoveryMapper, traitdata []byte, ioStreams cmdutil.IOStreams, cap *types.Capability) error { - var td v1beta1.TraitDefinition - var err error - if err = yaml.Unmarshal(traitdata, &td); err != nil { - return err - } - td.Namespace = types.DefaultKubeVelaNS - ioStreams.Info("Installing trait capability " + td.Name) - if cap.Install != nil { - cap.Source.ChartName = cap.Install.Helm.Name - if err = helm.InstallHelmChart(ioStreams, cap.Install.Helm); err != nil { - return err - } - err = addSourceIntoExtension(td.Spec.Extension, cap.Source) - if err != nil { - return err - } - } - if err = HackForStandardTrait(*cap, client); err != nil { - return err - } - gvk, err := util.GetGVKFromDefinition(mapper, td.Spec.Reference) - if err != nil { - return err - } - cap.CrdInfo = &types.CRDInfo{ - APIVersion: v1.GroupVersion{ - Group: gvk.Group, - Version: gvk.Version, - }.String(), - Kind: gvk.Kind, - } - if err = client.Create(context.Background(), &td); err != nil && !apierrors.IsAlreadyExists(err) { - return err - } - return nil -} - -// HackForStandardTrait will do some hack install for standard capability -func HackForStandardTrait(tp types.Capability, client client.Client) error { - switch tp.Name { - case "metrics": - // metrics trait will rely on a Prometheus instance to be installed - // make sure the chart is a prometheus operator - if tp.Install == nil { - break - } - if tp.Install.Helm.Namespace == "monitoring" && tp.Install.Helm.Name == "kube-prometheus-stack" { - if err := InstallPrometheusInstance(client); err != nil { - return err - } - } - default: - } - return nil -} - -// GetCapabilityFromCenter will list all synced capabilities from cap center and return the specified one -func GetCapabilityFromCenter(mapper discoverymapper.DiscoveryMapper, repoName, addonName string) (types.Capability, error) { - dir, _ := system.GetCapCenterDir() - repoDir := filepath.Join(dir, repoName) - templates, err := plugins.LoadCapabilityFromSyncedCenter(mapper, repoDir) - if err != nil { - return types.Capability{}, err - } - for _, t := range templates { - if t.Name == addonName { - t.Source = &types.Source{RepoName: repoName} - return t, nil - } - } - return types.Capability{}, fmt.Errorf("%s/%s not exist, try 'vela cap center sync %s' to sync from remote", repoName, addonName, repoName) -} - -// ListCapabilityCenters will list all capabilities from center -func ListCapabilityCenters() ([]apis.CapabilityCenterMeta, error) { - var capabilityCenterList []apis.CapabilityCenterMeta - centers, err := plugins.LoadRepos() - if err != nil { - return capabilityCenterList, err - } - for _, c := range centers { - capabilityCenterList = append(capabilityCenterList, apis.CapabilityCenterMeta{ - Name: c.Name, - URL: c.Address, - }) - } - return capabilityCenterList, nil -} - -// SyncCapabilityCenter will sync capabilities from center to local -func SyncCapabilityCenter(capabilityCenterName string) error { - repos, err := plugins.LoadRepos() - if err != nil { - return err - } - if len(repos) == 0 { - return fmt.Errorf("no capability center configured") - } - find := false - if capabilityCenterName != "" { - for idx, r := range repos { - if r.Name == capabilityCenterName { - repos = []plugins.CapCenterConfig{repos[idx]} - find = true - break - } - } - if !find { - return fmt.Errorf("%s center not exist", capabilityCenterName) - } - } - ctx := context.Background() - for _, d := range repos { - client, err := plugins.NewCenterClient(ctx, d.Name, d.Address, d.Token) - if err != nil { - return err - } - err = client.SyncCapabilityFromCenter() - if err != nil { - return err - } - } - return nil -} - -// RemoveCapabilityFromCluster will remove a capability from cluster. -// 1. remove definition 2. uninstall chart 3. remove local files -func RemoveCapabilityFromCluster(userNamespace string, c common.Args, client client.Client, capabilityName string) (string, error) { - ioStreams := cmdutil.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} - if err := RemoveCapability(userNamespace, c, client, capabilityName, ioStreams); err != nil { - return "", err - } - msg := fmt.Sprintf("%s removed successfully", capabilityName) - return msg, nil -} - -// RemoveCapability will remove a capability from cluster. -// 1. remove definition 2. uninstall chart 3. remove local files -func RemoveCapability(userNamespace string, c common.Args, client client.Client, capabilityName string, ioStreams cmdutil.IOStreams) error { - // TODO(wonderflow): make sure no apps is using this capability - caps, err := plugins.LoadAllInstalledCapability(userNamespace, c) - if err != nil { - return err - } - for _, w := range caps { - if w.Name == capabilityName { - return uninstallCap(client, w, ioStreams) - } - } - return errors.New(capabilityName + " not exist") -} - -func uninstallCap(cli client.Client, cap types.Capability, ioStreams cmdutil.IOStreams) error { - // 1. Remove WorkloadDefinition or TraitDefinition - ctx := context.Background() - var obj client.Object - switch cap.Type { - case types.TypeTrait: - obj = &v1beta1.TraitDefinition{ObjectMeta: v1.ObjectMeta{Name: cap.Name, Namespace: types.DefaultKubeVelaNS}} - case types.TypeWorkload: - obj = &v1beta1.WorkloadDefinition{ObjectMeta: v1.ObjectMeta{Name: cap.Name, Namespace: types.DefaultKubeVelaNS}} - case types.TypeScope: - return fmt.Errorf("uninstall scope capability was not supported yet") - case types.TypeComponentDefinition: - obj = &v1beta1.ComponentDefinition{ObjectMeta: v1.ObjectMeta{Name: cap.Name, Namespace: types.DefaultKubeVelaNS}} - default: - return fmt.Errorf("unsupported type: %v", cap.Type) - } - if err := cli.Delete(ctx, obj); err != nil { - return err - } - - if cap.Install != nil && cap.Install.Helm.Name != "" { - // 2. Remove Helm chart if there is - if cap.Install.Helm.Namespace == "" { - cap.Install.Helm.Namespace = types.DefaultKubeVelaNS - } - if err := helm.Uninstall(ioStreams, cap.Install.Helm.Name, cap.Install.Helm.Namespace, cap.Name); err != nil { - return err - } - } - - // 3. Remove local capability file - capdir, _ := system.GetCapabilityDir() - switch cap.Type { - case types.TypeTrait: - if err := os.Remove(filepath.Join(capdir, "traits", cap.Name)); err != nil { - return err - } - case types.TypeWorkload: - if err := os.Remove(filepath.Join(capdir, "workloads", cap.Name)); err != nil { - return err - } - case types.TypeScope: - // TODO(wonderflow): add scope remove here. - case types.TypeComponentDefinition: - if err := os.Remove(filepath.Join(capdir, "components", cap.Name)); err != nil { - return err - } - default: - return fmt.Errorf("unsupported type: %v", cap.Type) - } - ioStreams.Infof("Successfully uninstalled capability %s", cap.Name) - return nil -} - -// ListCapabilities will list all caps from specified center -func ListCapabilities(userNamespace string, c common.Args, capabilityCenterName string) ([]types.Capability, error) { - var capabilityList []types.Capability - dir, err := system.GetCapCenterDir() - if err != nil { - return capabilityList, err - } - if capabilityCenterName != "" { - return listCenterCapabilities(userNamespace, c, filepath.Join(dir, capabilityCenterName)) - } - dirs, err := os.ReadDir(dir) - if err != nil { - return capabilityList, err - } - for _, dd := range dirs { - if !dd.IsDir() { - continue - } - caps, err := listCenterCapabilities(userNamespace, c, filepath.Join(dir, dd.Name())) - if err != nil { - return capabilityList, err - } - capabilityList = append(capabilityList, caps...) - } - return capabilityList, nil -} -func listCenterCapabilities(userNamespace string, c common.Args, repoDir string) ([]types.Capability, error) { - dm, err := c.GetDiscoveryMapper() - if err != nil { - return nil, err - } - templates, err := plugins.LoadCapabilityFromSyncedCenter(dm, repoDir) - if err != nil { - return templates, err - } - if len(templates) < 1 { - return templates, nil - } - baseDir := filepath.Base(repoDir) - components := gatherComponents(userNamespace, c, templates) - for i, p := range templates { - status := checkInstallStatus(userNamespace, c, p) - convertedApplyTo := ConvertApplyTo(p.AppliesTo, components) - templates[i].Center = baseDir - templates[i].Status = status - templates[i].AppliesTo = convertedApplyTo - } - return templates, nil -} - -// RemoveCapabilityCenter will remove a cap center from local -func RemoveCapabilityCenter(centerName string) (string, error) { - var message string - var err error - dir, _ := system.GetCapCenterDir() - repoDir := filepath.Join(dir, centerName) - // 1.remove capability center dir - if _, err := os.Stat(repoDir); err != nil { - if os.IsNotExist(err) { - err = fmt.Errorf("%s capability center has not successfully synced", centerName) - return message, err - } - } - if err = os.RemoveAll(repoDir); err != nil { - return message, err - } - // 2.remove center from capability center config - repos, err := plugins.LoadRepos() - if err != nil { - return message, err - } - for idx, r := range repos { - if r.Name == centerName { - repos = append(repos[:idx], repos[idx+1:]...) - break - } - } - if err = plugins.StoreRepos(repos); err != nil { - return message, err - } - message = fmt.Sprintf("%s capability center removed successfully", centerName) - return message, err -} - -func gatherComponents(userNamespace string, c common.Args, templates []types.Capability) []types.Capability { - components, err := plugins.LoadInstalledCapabilityWithType(userNamespace, c, types.TypeComponentDefinition) - if err != nil { - components = make([]types.Capability, 0) - } - for _, t := range templates { - if t.Type == types.TypeComponentDefinition { - components = append(components, t) - } - } - return components -} - -func checkInstallStatus(userNamespace string, c common.Args, tmp types.Capability) string { - var status = "uninstalled" - installed, _ := plugins.LoadInstalledCapabilityWithType(userNamespace, c, tmp.Type) - for _, i := range installed { - if i.Name == tmp.Name && i.CrdName == tmp.CrdName { - return "installed" - } - } - return status -} - -func addSourceIntoExtension(in *runtime.RawExtension, source *types.Source) error { - var extension map[string]interface{} - err := json.Unmarshal(in.Raw, &extension) - if err != nil { - return err - } - extension["source"] = source - data, err := json.Marshal(extension) - if err != nil { - return err - } - in.Raw = data - return nil -} - -// GetCapabilityConfigMap gets the ConfigMap which stores the information of a capability -func GetCapabilityConfigMap(kubeClient client.Client, capabilityName string) (corev1.ConfigMap, error) { - cmName := fmt.Sprintf("%s%s", types.CapabilityConfigMapNamePrefix, capabilityName) - var cm corev1.ConfigMap - err := kubeClient.Get(context.Background(), client.ObjectKey{Namespace: types.DefaultKubeVelaNS, Name: cmName}, &cm) - return cm, err -} - -// CheckLabelExistence checks whether a label `key=value` exists in definition labels -func CheckLabelExistence(labels map[string]string, label string) bool { - splitLabel := strings.Split(label, "=") - k, v := splitLabel[0], splitLabel[1] - if labelValue, ok := labels[k]; ok { - if labelValue == v { - return true - } - } - return false -} diff --git a/references/common/component.go b/references/common/component.go deleted file mode 100644 index b64161aaa..000000000 --- a/references/common/component.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package common - -import ( - "context" - - "github.com/oam-dev/kubevela/references/apis" - - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// RetrieveComponent will get component status -func RetrieveComponent(ctx context.Context, c client.Reader, applicationName, componentName, - namespace string) (apis.ComponentMeta, error) { - var componentMeta apis.ComponentMeta - applicationMeta, err := RetrieveApplicationStatusByName(ctx, c, applicationName, namespace) - if err != nil { - return componentMeta, err - } - - for _, com := range applicationMeta.Components { - if com.Name != componentName { - continue - } - return com, nil - } - return componentMeta, nil -} diff --git a/references/common/registry.go b/references/common/registry.go new file mode 100644 index 000000000..85d4824d5 --- /dev/null +++ b/references/common/registry.go @@ -0,0 +1,159 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "context" + "encoding/json" + "strings" + + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/apis/types" + "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" + "github.com/oam-dev/kubevela/pkg/oam/util" + "github.com/oam-dev/kubevela/pkg/utils/helm" + cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" +) + +// InstallComponentDefinition will add a component into K8s cluster and install its controller +func InstallComponentDefinition(client client.Client, componentData []byte, ioStreams cmdutil.IOStreams, tp *types.Capability) error { + var cd v1beta1.ComponentDefinition + var err error + if componentData == nil { + return errors.New("componentData is nil") + } + if err = yaml.Unmarshal(componentData, &cd); err != nil { + return err + } + cd.Namespace = types.DefaultKubeVelaNS + ioStreams.Info("Installing component: " + cd.Name) + if tp.Install != nil { + tp.Source.ChartName = tp.Install.Helm.Name + if err = helm.InstallHelmChart(ioStreams, tp.Install.Helm); err != nil { + return err + } + err = addSourceIntoExtension(cd.Spec.Extension, tp.Source) + if err != nil { + return err + } + } + if cd.Spec.Workload.Type == "" { + tp.CrdInfo = &types.CRDInfo{ + APIVersion: cd.Spec.Workload.Definition.APIVersion, + Kind: cd.Spec.Workload.Definition.Kind, + } + } + if err = client.Create(context.Background(), &cd); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} + +// InstallTraitDefinition will add a trait into K8s cluster and install it's controller +func InstallTraitDefinition(client client.Client, mapper discoverymapper.DiscoveryMapper, traitdata []byte, ioStreams cmdutil.IOStreams, cap *types.Capability) error { + var td v1beta1.TraitDefinition + var err error + if err = yaml.Unmarshal(traitdata, &td); err != nil { + return err + } + td.Namespace = types.DefaultKubeVelaNS + ioStreams.Info("Installing trait " + td.Name) + if cap.Install != nil { + cap.Source.ChartName = cap.Install.Helm.Name + if err = helm.InstallHelmChart(ioStreams, cap.Install.Helm); err != nil { + return err + } + err = addSourceIntoExtension(td.Spec.Extension, cap.Source) + if err != nil { + return err + } + } + if err = HackForStandardTrait(*cap, client); err != nil { + return err + } + gvk, err := util.GetGVKFromDefinition(mapper, td.Spec.Reference) + if err != nil { + return err + } + cap.CrdInfo = &types.CRDInfo{ + APIVersion: v1.GroupVersion{ + Group: gvk.Group, + Version: gvk.Version, + }.String(), + Kind: gvk.Kind, + } + if err = client.Create(context.Background(), &td); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} + +// HackForStandardTrait will do some hack install for standard registry +func HackForStandardTrait(tp types.Capability, client client.Client) error { + switch tp.Name { + case "metrics": + // metrics trait will rely on a Prometheus instance to be installed + // make sure the chart is a prometheus operator + if tp.Install == nil { + break + } + if tp.Install.Helm.Namespace == "monitoring" && tp.Install.Helm.Name == "kube-prometheus-stack" { + if err := InstallPrometheusInstance(client); err != nil { + return err + } + } + default: + } + return nil +} + +func addSourceIntoExtension(in *runtime.RawExtension, source *types.Source) error { + var extension map[string]interface{} + err := json.Unmarshal(in.Raw, &extension) + if err != nil { + return err + } + extension["source"] = source + data, err := json.Marshal(extension) + if err != nil { + return err + } + in.Raw = data + return nil +} + +// CheckLabelExistence checks whether a label `key=value` exists in definition labels +func CheckLabelExistence(labels map[string]string, label string) bool { + splitLabel := strings.Split(label, "=") + if len(splitLabel) < 2 { + return false + } + k, v := splitLabel[0], splitLabel[1] + if labelValue, ok := labels[k]; ok { + if labelValue == v { + return true + } + } + return false +} diff --git a/references/common/capability_test.go b/references/common/registry_test.go similarity index 100% rename from references/common/capability_test.go rename to references/common/registry_test.go diff --git a/references/common/trait.go b/references/common/trait.go index f26a298ff..8daf736cd 100644 --- a/references/common/trait.go +++ b/references/common/trait.go @@ -18,7 +18,6 @@ package common import ( "context" - "fmt" "strings" plur "github.com/gertd/go-pluralize" @@ -29,46 +28,8 @@ import ( "github.com/oam-dev/kubevela/pkg/oam" "github.com/oam-dev/kubevela/pkg/oam/util" "github.com/oam-dev/kubevela/pkg/utils/common" - "github.com/oam-dev/kubevela/references/plugins" ) -// ListTraitDefinitions will list all definition include traits and workloads -func ListTraitDefinitions(userNamespace string, c common.Args, workloadName *string) ([]types.Capability, error) { - var traitList []types.Capability - traits, err := plugins.LoadInstalledCapabilityWithType(userNamespace, c, types.TypeTrait) - if err != nil { - return traitList, err - } - workloads, err := plugins.LoadInstalledCapabilityWithType(userNamespace, c, types.TypeComponentDefinition) - if err != nil { - return traitList, err - } - traitList = convertAllApplyToList(traits, workloads, workloadName) - return traitList, nil -} - -// ListRawTraitDefinitions will list raw definition -func ListRawTraitDefinitions(userNamespace string, c common.Args) ([]v1beta1.TraitDefinition, error) { - client, err := c.GetClient() - if err != nil { - return nil, err - } - ctx := util.SetNamespaceInCtx(context.Background(), userNamespace) - traitList := v1beta1.TraitDefinitionList{} - ns := ctx.Value(util.AppDefinitionNamespace).(string) - if err = client.List(ctx, &traitList, client2.InNamespace(ns)); err != nil { - return nil, err - } - if ns == oam.SystemDefinitonNamespace { - return traitList.Items, nil - } - sysTraitList := v1beta1.TraitDefinitionList{} - if err = client.List(ctx, &sysTraitList, client2.InNamespace(oam.SystemDefinitonNamespace)); err != nil { - return nil, err - } - return append(traitList.Items, sysTraitList.Items...), nil -} - // ListRawWorkloadDefinitions will list raw definition func ListRawWorkloadDefinitions(userNamespace string, c common.Args) ([]v1beta1.WorkloadDefinition, error) { client, err := c.GetClient() @@ -91,63 +52,6 @@ func ListRawWorkloadDefinitions(userNamespace string, c common.Args) ([]v1beta1. return append(workloadList.Items, sysWorkloadList.Items...), nil } -// ListRawComponentDefinitions will list raw component definition -func ListRawComponentDefinitions(userNamespace string, c common.Args) ([]v1beta1.ComponentDefinition, error) { - client, err := c.GetClient() - if err != nil { - return nil, err - } - ctx := util.SetNamespaceInCtx(context.Background(), userNamespace) - ns := ctx.Value(util.AppDefinitionNamespace).(string) - componentList := v1beta1.ComponentDefinitionList{} - if err = client.List(ctx, &componentList, client2.InNamespace(ns)); err != nil { - return nil, err - } - if ns == oam.SystemDefinitonNamespace { - return componentList.Items, nil - } - sysComponentList := v1beta1.ComponentDefinitionList{} - if err = client.List(ctx, &sysComponentList, client2.InNamespace(oam.SystemDefinitonNamespace)); err != nil { - return nil, err - } - return append(componentList.Items, sysComponentList.Items...), nil -} - -// GetTraitDefinition will get trait capability with applyTo converted -func GetTraitDefinition(userNamespace string, c common.Args, workloadName *string, traitType string) (types.Capability, error) { - var traitDef types.Capability - traitCap, err := plugins.GetInstalledCapabilityWithCapName(types.TypeTrait, traitType) - if err != nil { - return traitDef, err - } - workloadsCap, err := plugins.LoadInstalledCapabilityWithType(userNamespace, c, types.TypeComponentDefinition) - if err != nil { - return traitDef, err - } - traitList := convertAllApplyToList([]types.Capability{traitCap}, workloadsCap, workloadName) - if len(traitList) != 1 { - return traitDef, fmt.Errorf("could not get installed capability by %s", traitType) - } - traitDef = traitList[0] - return traitDef, nil -} - -func convertAllApplyToList(traits []types.Capability, workloads []types.Capability, workloadName *string) []types.Capability { - var traitList []types.Capability - for _, t := range traits { - convertedApplyTo := ConvertApplyTo(t.AppliesTo, workloads) - if *workloadName != "" { - if !in(convertedApplyTo, *workloadName) { - continue - } - convertedApplyTo = []string{*workloadName} - } - t.AppliesTo = convertedApplyTo - traitList = append(traitList, t) - } - return traitList -} - // ConvertApplyTo will convert applyTo slice to workload capability name if CRD matches func ConvertApplyTo(applyTo []string, workloads []types.Capability) []string { var converted []string diff --git a/references/plugins/capcenter.go b/references/plugins/capcenter.go deleted file mode 100644 index 01c4038ac..000000000 --- a/references/plugins/capcenter.go +++ /dev/null @@ -1,344 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package plugins - -import ( - "context" - "errors" - "fmt" - "net/http" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/google/go-github/v32/github" - "golang.org/x/oauth2" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "sigs.k8s.io/yaml" - - "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" - "github.com/oam-dev/kubevela/apis/types" - "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" - "github.com/oam-dev/kubevela/pkg/oam/util" - "github.com/oam-dev/kubevela/pkg/utils/system" -) - -// Content contains different type of content needed when building Registry -type Content struct { - OssContent - GithubContent - LocalContent -} - -// LocalContent for local registry -type LocalContent struct { - AbsDir string `json:"abs_dir"` -} - -// OssContent for oss registry -type OssContent struct { - BucketURL string `json:"bucket_url"` -} - -// GithubContent for cap center -type GithubContent struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - Path string `json:"path"` - Ref string `json:"ref"` -} - -// CapCenterConfig is used to store cap center config in file -type CapCenterConfig struct { - Name string `json:"name"` - Address string `json:"address"` - Token string `json:"token"` -} - -// CenterClient defines an interface for cap center client -type CenterClient interface { - SyncCapabilityFromCenter() error -} - -// NewCenterClient create a client from type -func NewCenterClient(ctx context.Context, name, address, token string) (CenterClient, error) { - Type, cfg, err := Parse(address) - if err != nil { - return nil, err - } - switch Type { - case TypeGithub: - return NewGithubCenter(ctx, token, name, &cfg.GithubContent) - case TypeOss: - return NewOssCenter(fmt.Sprintf("https://%s/", cfg.BucketURL), name), nil - default: - } - return nil, errors.New("we only support github as repository now") -} - -// TypeLocal represents github -const TypeLocal = "local" - -// TypeOss represent oss -const TypeOss = "oss" - -// TypeGithub represents github -const TypeGithub = "github" - -// TypeUnknown represents parse failed -const TypeUnknown = "unknown" - -// Parse will parse config from address -func Parse(addr string) (string, *Content, error) { - URL, err := url.Parse(addr) - if err != nil { - return "", nil, err - } - l := strings.Split(strings.TrimPrefix(URL.Path, "/"), "/") - switch URL.Scheme { - case "http", "https": - switch URL.Host { - case "github.com": - // We support two valid format: - // 1. https://github.com///tree// - // 2. https://github.com/// - if len(l) < 3 { - return "", nil, errors.New("invalid format " + addr) - } - if l[2] == "tree" { - // https://github.com///tree// - if len(l) < 5 { - return "", nil, errors.New("invalid format " + addr) - } - return TypeGithub, &Content{ - GithubContent: GithubContent{ - Owner: l[0], - Repo: l[1], - Path: strings.Join(l[4:], "/"), - Ref: l[3], - }, - }, nil - } - // https://github.com/// - return TypeGithub, &Content{ - GithubContent: GithubContent{ - Owner: l[0], - Repo: l[1], - Path: strings.Join(l[2:], "/"), - Ref: "", // use default branch - }, - }, - nil - case "api.github.com": - if len(l) != 5 { - return "", nil, errors.New("invalid format " + addr) - } - //https://api.github.com/repos///contents/ - return TypeGithub, &Content{ - GithubContent: GithubContent{ - Owner: l[1], - Repo: l[2], - Path: l[4], - Ref: URL.Query().Get("ref"), - }, - }, - nil - default: - } - case "oss": - return TypeOss, &Content{ - OssContent: OssContent{ - BucketURL: URL.Host, - }, - }, nil - case "file": - return TypeLocal, &Content{ - LocalContent: LocalContent{ - AbsDir: URL.Path, - }, - }, nil - - } - - return TypeUnknown, nil, nil -} - -// LoadRepos will load all cap center repos -func LoadRepos() ([]CapCenterConfig, error) { - defaultRepo := CapCenterConfig{ - Name: "default-cap-center", - Address: "oss://registry.kubevela.net/", - } - config, err := system.GetRepoConfig() - if err != nil { - return nil, err - } - data, err := os.ReadFile(filepath.Clean(config)) - if err != nil { - if os.IsNotExist(err) { - return []CapCenterConfig{defaultRepo}, nil - } - return nil, err - } - var repos []CapCenterConfig - if err = yaml.Unmarshal(data, &repos); err != nil { - return nil, err - } - haveDefault := false - for _, repo := range repos { - if repo.Address == defaultRepo.Address { - haveDefault = true - break - } - } - if !haveDefault { - repos = append(repos, defaultRepo) - } - return repos, nil -} - -// StoreRepos will store cap center repo locally -func StoreRepos(repos []CapCenterConfig) error { - config, err := system.GetRepoConfig() - if err != nil { - return err - } - data, err := yaml.Marshal(repos) - if err != nil { - return err - } - //nolint:gosec - return os.WriteFile(config, data, 0644) -} - -// ParseCapability will convert config from remote center to capability -func ParseCapability(mapper discoverymapper.DiscoveryMapper, data []byte) (types.Capability, error) { - var obj = unstructured.Unstructured{Object: make(map[string]interface{})} - err := yaml.Unmarshal(data, &obj.Object) - if err != nil { - return types.Capability{}, err - } - switch obj.GetKind() { - case "ComponentDefinition": - var cd v1beta1.ComponentDefinition - err = yaml.Unmarshal(data, &cd) - if err != nil { - return types.Capability{}, err - } - ref, err := util.ConvertWorkloadGVK2Definition(mapper, cd.Spec.Workload.Definition) - if err != nil { - return types.Capability{}, err - } - return HandleDefinition(cd.Name, ref.Name, cd.Annotations, cd.Labels, cd.Spec.Extension, types.TypeComponentDefinition, nil, cd.Spec.Schematic) - case "TraitDefinition": - var td v1beta1.TraitDefinition - err = yaml.Unmarshal(data, &td) - if err != nil { - return types.Capability{}, err - } - return HandleDefinition(td.Name, td.Spec.Reference.Name, td.Annotations, td.Labels, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads, td.Spec.Schematic) - case "ScopeDefinition": - // TODO(wonderflow): support scope definition here. - } - return types.Capability{}, fmt.Errorf("unknown definition Type %s", obj.GetKind()) -} - -// NewGithubCenter will create client by github center implementation -func NewGithubCenter(ctx context.Context, token, centerName string, r *GithubContent) (*GithubRegistry, error) { - var tc *http.Client - if token != "" { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc = oauth2.NewClient(ctx, ts) - } - return &GithubRegistry{client: github.NewClient(tc), cfg: r, centerName: centerName, ctx: ctx}, nil -} - -// SyncCapabilityFromCenter will sync capability from github registry -// TODO(wonderflow): currently we only sync by create, we also need to delete which not exist remotely. -func (g *GithubRegistry) SyncCapabilityFromCenter() error { - dir, err := system.GetCapCenterDir() - if err != nil { - return err - } - repoDir := filepath.Join(dir, g.centerName) - _, _ = system.CreateIfNotExist(repoDir) - var success int - items, err := g.getRepoFile() - if err != nil { - return err - } - for _, item := range items { - addon, err := item.toAddon() - if err != nil { - fmt.Printf("[INFO] CRD for %s not found\n", item.name) - continue - } - //nolint:gosec - err = os.WriteFile(filepath.Join(repoDir, addon.Name+".yaml"), item.data, 0644) - if err != nil { - fmt.Printf("write definition %s to %s err %v\n", addon.Name+".yaml", repoDir, err) - continue - } - success++ - } - fmt.Printf("successfully sync %d from %s remote center\n", success, g.centerName) - return nil -} - -// NewOssCenter will create OSS center implementation -func NewOssCenter(bucketURL string, centerName string) *OssRegistry { - var tc http.Client - return &OssRegistry{ - Client: &tc, - bucketURL: bucketURL, - centerName: centerName, - } -} - -// SyncCapabilityFromCenter will sync capability from oss registry -func (o *OssRegistry) SyncCapabilityFromCenter() error { - dir, err := system.GetCapCenterDir() - if err != nil { - return err - } - repoDir := filepath.Join(dir, o.centerName) - _, _ = system.CreateIfNotExist(repoDir) - var success int - items, err := o.getRegFiles() - if err != nil { - return err - } - for _, item := range items { - addon, err := item.toAddon() - if err != nil { - fmt.Printf("[INFO] CRD for %s not found\n", item.name) - continue - } - //nolint:gosec - err = os.WriteFile(filepath.Join(repoDir, addon.Name+".yaml"), item.data, 0644) - if err != nil { - fmt.Printf("write definition %s to %s err %v\n", addon.Name+".yaml", repoDir, err) - continue - } - success++ - } - fmt.Printf("successfully sync %d from %s remote center\n", success, o.centerName) - return nil -} diff --git a/references/plugins/capcenter_test.go b/references/plugins/capcenter_test.go deleted file mode 100644 index 3a53047a5..000000000 --- a/references/plugins/capcenter_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package plugins - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestParseURL(t *testing.T) { - cases := map[string]struct { - url string - exp *GithubContent - expType string - }{ - "api-github": { - url: "https://api.github.com/repos/zzxwill/catalog/contents/repository?ref=plugin", - expType: TypeGithub, - exp: &GithubContent{ - Owner: "zzxwill", - Repo: "catalog", - Path: "repository", - Ref: "plugin", - }, - }, - "github-copy-path": { - url: "https://github.com/zzxwill/catalog/tree/plugin/repository", - expType: TypeGithub, - exp: &GithubContent{ - Owner: "zzxwill", - Repo: "catalog", - Path: "repository", - Ref: "plugin", - }, - }, - "github-manuel-write-path": { - url: "https://github.com/zzxwill/catalog/repository", - expType: TypeGithub, - exp: &GithubContent{ - Owner: "zzxwill", - Repo: "catalog", - Path: "repository", - }, - }, - } - for caseName, c := range cases { - tp, content, err := Parse(c.url) - assert.NoError(t, err, caseName) - assert.Equal(t, c.exp, &content.GithubContent, caseName) - assert.Equal(t, c.expType, tp, caseName) - } -} diff --git a/references/plugins/local.go b/references/plugins/local.go index d2a0a8c11..150487156 100644 --- a/references/plugins/local.go +++ b/references/plugins/local.go @@ -17,18 +17,11 @@ limitations under the License. package plugins import ( - "bytes" "context" - "encoding/json" "fmt" - "os" - "path/filepath" - "strings" "github.com/oam-dev/kubevela/apis/types" - "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" "github.com/oam-dev/kubevela/pkg/utils/common" - "github.com/oam-dev/kubevela/pkg/utils/system" ) // LoadCapabilityByName will load capability from local by name @@ -94,176 +87,3 @@ func LoadInstalledCapabilityWithType(userNamespace string, c common.Args, capT t } return nil, nil } - -// GetInstalledCapabilityWithCapName will get cap by alias -func GetInstalledCapabilityWithCapName(capT types.CapType, capName string) (types.Capability, error) { - dir, err := system.GetCapabilityDir() - if err != nil { - return types.Capability{}, err - } - return loadInstalledCapabilityWithCapName(dir, capT, capName) -} - -// leave dir as argument for test convenience -func loadInstalledCapabilityWithType(dir string, capT types.CapType) ([]types.Capability, error) { - dir = GetSubDir(dir, capT) - return loadInstalledCapability(dir, "") -} - -func loadInstalledCapabilityWithCapName(dir string, capT types.CapType, capName string) (types.Capability, error) { - var cap types.Capability - dir = GetSubDir(dir, capT) - capList, err := loadInstalledCapability(dir, capName) - if err != nil { - return cap, err - } else if len(capList) != 1 { - return cap, fmt.Errorf("could not get installed capability by %s", capName) - } - return capList[0], nil -} - -func loadInstalledCapability(dir string, name string) ([]types.Capability, error) { - var tmps []types.Capability - files, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - for _, f := range files { - if f.IsDir() { - continue - } - if strings.HasSuffix(f.Name(), ".cue") { - continue - } - data, err := os.ReadFile(filepath.Clean(filepath.Join(dir, f.Name()))) - if err != nil { - fmt.Printf("read file %s err %v\n", f.Name(), err) - continue - } - var tmp types.Capability - decoder := json.NewDecoder(bytes.NewBuffer(data)) - decoder.UseNumber() - if err = decoder.Decode(&tmp); err != nil { - fmt.Printf("ignore invalid format file: %s\n", f.Name()) - continue - } - // Get the specified installed capability: workload or trait - if name != "" { - if name == f.Name() { - tmps = append(tmps, tmp) - break - } - continue - } - tmps = append(tmps, tmp) - } - return tmps, nil -} - -// GetSubDir will get dir for capability -func GetSubDir(dir string, capT types.CapType) string { - switch capT { - case types.TypeWorkload: - return filepath.Join(dir, "workloads") - case types.TypeTrait: - return filepath.Join(dir, "traits") - case types.TypeScope: - return filepath.Join(dir, "scopes") - case types.TypeComponentDefinition: - return filepath.Join(dir, "components") - default: - } - return dir -} - -// SinkTemp2Local will sink template to local file -func SinkTemp2Local(templates []types.Capability, dir string) int { - success := 0 - for _, tmp := range templates { - subDir := GetSubDir(dir, tmp.Type) - _, _ = system.CreateIfNotExist(subDir) - data, err := json.Marshal(tmp) - if err != nil { - fmt.Printf("sync %s err: %v\n", tmp.Name, err) - continue - } - //nolint:gosec - err = os.WriteFile(filepath.Join(subDir, tmp.Name), data, 0644) - if err != nil { - fmt.Printf("sync %s err: %v\n", tmp.Name, err) - continue - } - success++ - } - return success -} - -// RemoveLegacyTemps will remove capability definitions under `dir` but not included in `retainedTemps`. -func RemoveLegacyTemps(retainedTemps []types.Capability, dir string) int { - success := 0 - var retainedFiles []string - subDirs := []string{GetSubDir(dir, types.TypeComponentDefinition), GetSubDir(dir, types.TypeTrait)} - for _, tmp := range retainedTemps { - subDir := GetSubDir(dir, tmp.Type) - tmpFilePath := filepath.Join(subDir, tmp.Name) - retainedFiles = append(retainedFiles, tmpFilePath) - } - - for _, subDir := range subDirs { - if err := filepath.Walk(subDir, func(path string, info os.FileInfo, err error) error { - if info == nil || info.IsDir() { - // omit subDir or subDir not exist - return nil - } - for _, retainedFile := range retainedFiles { - if retainedFile == path { - return nil - } - } - if err := os.Remove(path); err != nil { - fmt.Printf("remove legacy %s err: %v\n", path, err) - return err - } - success++ - return nil - }); err != nil { - continue - } - } - return success -} - -// LoadCapabilityFromSyncedCenter will load capability from dir -func LoadCapabilityFromSyncedCenter(mapper discoverymapper.DiscoveryMapper, dir string) ([]types.Capability, error) { - var tmps []types.Capability - files, err := os.ReadDir(dir) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - for _, f := range files { - if f.IsDir() { - continue - } - if strings.HasSuffix(f.Name(), ".cue") { - continue - } - data, err := os.ReadFile(filepath.Clean(filepath.Join(dir, f.Name()))) - if err != nil { - fmt.Printf("read file %s err %v\n", f.Name(), err) - continue - } - tmp, err := ParseCapability(mapper, data) - if err != nil { - fmt.Printf("get definition of %s err %v\n", f.Name(), err) - continue - } - tmps = append(tmps, tmp) - } - return tmps, nil -} diff --git a/references/plugins/local_test.go b/references/plugins/local_test.go index 52b0cd709..ca6f8e691 100644 --- a/references/plugins/local_test.go +++ b/references/plugins/local_test.go @@ -17,12 +17,7 @@ limitations under the License. package plugins import ( - "os" - "testing" - "github.com/oam-dev/kubevela/apis/types" - - "github.com/stretchr/testify/assert" ) var ( @@ -60,105 +55,3 @@ var ( }, } ) - -func TestLocalSink(t *testing.T) { - - cases := map[string]struct { - dir string - tmps []types.Capability - Type types.CapType - expDef []types.Capability - err error - }{ - "Test No Templates": { - dir: "vela-test1", - tmps: nil, - }, - "Test Only Workload": { - dir: "vela-test2", - tmps: []types.Capability{deployment, statefulset}, - Type: types.TypeComponentDefinition, - expDef: []types.Capability{deployment, statefulset}, - }, - "Test Only Trait": { - dir: "vela-test3", - tmps: []types.Capability{route}, - Type: types.TypeTrait, - expDef: []types.Capability{route}, - }, - "Test Only Workload But want trait": { - dir: "vela-test3", - tmps: []types.Capability{deployment, statefulset}, - Type: types.TypeTrait, - expDef: nil, - }, - "Test Both have Workload and trait But want Workload": { - dir: "vela-test4", - tmps: []types.Capability{deployment, route, statefulset}, - Type: types.TypeComponentDefinition, - expDef: []types.Capability{deployment, statefulset}, - }, - "Test Both have Workload and trait But want Trait": { - dir: "vela-test5", - tmps: []types.Capability{deployment, route, statefulset}, - Type: types.TypeTrait, - expDef: []types.Capability{route}, - }, - } - for name, c := range cases { - testInDir(t, name, c.dir, c.tmps, c.expDef, c.Type, c.err) - } -} - -func testInDir(t *testing.T, casename, dir string, tmps, defexp []types.Capability, Type types.CapType, err1 error) { - err := os.MkdirAll(dir, 0755) - assert.NoError(t, err, casename) - defer os.RemoveAll(dir) - number := SinkTemp2Local(tmps, dir) - assert.Equal(t, len(tmps), number) - if Type != "" { - gotDef, err := loadInstalledCapabilityWithType(dir, Type) - assert.NoError(t, err, casename) - assert.Equal(t, defexp, gotDef, casename) - } -} - -func TestRemoveLegacyTemps(t *testing.T) { - - cases := []struct { - caseName string - newTemps []types.Capability - rmNum int - }{ - { - caseName: "remove all", - newTemps: []types.Capability{}, - rmNum: 3, - }, - { - caseName: "nothing removed", - newTemps: []types.Capability{deployment, statefulset, route}, - rmNum: 0, - }, - { - caseName: "remove part of existings", - newTemps: []types.Capability{statefulset, route}, - rmNum: 1, - }, - } - for _, c := range cases { - runInDirRemoveLegacyTemps(t, c.caseName, c.newTemps, c.rmNum) - } -} - -func runInDirRemoveLegacyTemps(t *testing.T, caseName string, newTemps []types.Capability, rmNum int) { - dir := "vela-test-rm-temps" - err := os.MkdirAll(dir, 0755) - assert.NoError(t, err, caseName) - defer os.RemoveAll(dir) - existingTemps := []types.Capability{deployment, statefulset, route} - number := SinkTemp2Local(existingTemps, dir) - assert.Equal(t, 3, number) - resultRemoveNum := RemoveLegacyTemps(newTemps, dir) - assert.Equal(t, rmNum, resultRemoveNum, caseName) -} diff --git a/references/plugins/registry.go b/references/plugins/registry.go deleted file mode 100644 index f60eefcfa..000000000 --- a/references/plugins/registry.go +++ /dev/null @@ -1,332 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package plugins - -import ( - "context" - "encoding/base64" - "encoding/xml" - "fmt" - "io" - "net/http" - "os" - "path" - "path/filepath" - - "github.com/oam-dev/kubevela/apis/types" - - "github.com/google/go-github/v32/github" - "github.com/pkg/errors" - "golang.org/x/oauth2" - - "github.com/oam-dev/kubevela/pkg/utils/common" -) - -// Registry define a registry stores trait & component defs -type Registry interface { - GetCap(addonName string) (types.Capability, []byte, error) - ListCaps() ([]types.Capability, error) -} - -// GithubRegistry is Registry's implementation treat github url as resource -type GithubRegistry struct { - client *github.Client - cfg *GithubContent - ctx context.Context - centerName string // to be used to cache registry -} - -// NewRegistry will create a registry implementation -func NewRegistry(ctx context.Context, token, registryName string, regURL string) (Registry, error) { - tp, cfg, err := Parse(regURL) - if err != nil { - return nil, err - } - switch tp { - case TypeGithub: - var tc *http.Client - if token != "" { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc = oauth2.NewClient(ctx, ts) - } - return GithubRegistry{client: github.NewClient(tc), cfg: &cfg.GithubContent, ctx: ctx, centerName: registryName}, nil - case TypeOss: - var tc http.Client - return OssRegistry{ - Client: &tc, - bucketURL: fmt.Sprintf("https://%s/", cfg.BucketURL), - }, nil - case TypeLocal: - _, err := os.Stat(cfg.AbsDir) - if os.IsNotExist(err) { - return LocalRegistry{}, err - } - return LocalRegistry{absPath: cfg.AbsDir}, nil - case TypeUnknown: - return nil, fmt.Errorf("not supported url") - } - - return nil, fmt.Errorf("not supported url") -} - -// ListCaps list all capabilities of registry -func (g GithubRegistry) ListCaps() ([]types.Capability, error) { - var addons []types.Capability - - itemContents, err := g.getRepoFile() - if err != nil { - return []types.Capability{}, err - } - for _, item := range itemContents { - capa, err := item.toAddon() - if err != nil { - fmt.Printf("parse definition of %s err %v\n", item.name, err) - continue - } - addons = append(addons, capa) - } - return addons, nil -} - -// GetCap return capability object and raw data specified by cap name -func (g GithubRegistry) GetCap(addonName string) (types.Capability, []byte, error) { - fileContent, _, _, err := g.client.Repositories.GetContents(context.Background(), g.cfg.Owner, g.cfg.Repo, fmt.Sprintf("%s/%s.yaml", g.cfg.Path, addonName), &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) - if err != nil { - return types.Capability{}, []byte{}, err - } - var data []byte - if *fileContent.Encoding == "base64" { - data, err = base64.StdEncoding.DecodeString(*fileContent.Content) - if err != nil { - fmt.Printf("decode github content %s err %s\n", fileContent.GetPath(), err) - } - } - repoFile := RegistryFile{ - data: data, - name: *fileContent.Name, - } - addon, err := repoFile.toAddon() - if err != nil { - return types.Capability{}, []byte{}, err - } - return addon, data, nil -} - -func (g *GithubRegistry) getRepoFile() ([]RegistryFile, error) { - var items []RegistryFile - _, dirs, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, g.cfg.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) - if err != nil { - return []RegistryFile{}, err - } - for _, repoItem := range dirs { - if *repoItem.Type != "file" { - continue - } - fileContent, _, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, *repoItem.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) - if err != nil { - fmt.Printf("Getting content URL %s error: %s\n", repoItem.GetURL(), err) - continue - } - var data []byte - if *fileContent.Encoding == "base64" { - data, err = base64.StdEncoding.DecodeString(*fileContent.Content) - if err != nil { - fmt.Printf("decode github content %s err %s\n", fileContent.GetPath(), err) - continue - } - } - items = append(items, RegistryFile{ - data: data, - name: *fileContent.Name, - }) - } - return items, nil -} - -// OssRegistry is Registry's implementation treat OSS url as resource -type OssRegistry struct { - *http.Client - bucketURL string - centerName string -} - -// GetCap return capability object and raw data specified by cap name -func (o OssRegistry) GetCap(addonName string) (types.Capability, []byte, error) { - filename := addonName + ".yaml" - req, _ := http.NewRequestWithContext( - context.Background(), - http.MethodGet, - o.bucketURL+filename, - nil, - ) - resp, err := o.Client.Do(req) - if err != nil { - return types.Capability{}, nil, err - } - data, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if err != nil { - return types.Capability{}, nil, err - } - rf := RegistryFile{ - data: data, - name: filename, - } - capa, err := rf.toAddon() - if err != nil { - return types.Capability{}, nil, err - } - - return capa, data, nil -} - -// ListCaps list all capabilities of registry -func (o OssRegistry) ListCaps() ([]types.Capability, error) { - rfs, err := o.getRegFiles() - if err != nil { - return []types.Capability{}, errors.Wrap(err, "Get raw files fail") - } - capas := make([]types.Capability, 0) - - for _, rf := range rfs { - capa, err := rf.toAddon() - if err != nil { - fmt.Printf("[WARN] Parse file %s fail: %s\n", rf.name, err.Error()) - } - capas = append(capas, capa) - } - return capas, nil -} -func (o OssRegistry) getRegFiles() ([]RegistryFile, error) { - req, _ := http.NewRequestWithContext( - context.Background(), - http.MethodGet, - o.bucketURL+"?list-type=2", - nil, - ) - resp, err := o.Client.Do(req) - if err != nil { - return []RegistryFile{}, err - } - data, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() - if err != nil { - return []RegistryFile{}, err - } - list := &ListBucketResult{} - err = xml.Unmarshal(data, list) - if err != nil { - return []RegistryFile{}, err - } - rfs := make([]RegistryFile, 0) - - for _, fileName := range list.File { - req, _ := http.NewRequestWithContext( - context.Background(), - http.MethodGet, - o.bucketURL+fileName, - nil, - ) - resp, err := o.Client.Do(req) - if err != nil { - fmt.Printf("[WARN] %s download fail\n", fileName) - continue - } - data, _ := io.ReadAll(resp.Body) - _ = resp.Body.Close() - rf := RegistryFile{ - data: data, - name: fileName, - } - rfs = append(rfs, rf) - - } - return rfs, nil -} - -// LocalRegistry is Registry's implementation treat local url as resource -type LocalRegistry struct { - absPath string -} - -// GetCap return capability object and raw data specified by cap name -func (l LocalRegistry) GetCap(addonName string) (types.Capability, []byte, error) { - fileName := addonName + ".yaml" - filePath := fmt.Sprintf("%s/%s", l.absPath, fileName) - data, err := os.ReadFile(filePath) - if err != nil { - return types.Capability{}, []byte{}, err - } - file := RegistryFile{ - data: data, - name: fileName, - } - capa, err := file.toAddon() - if err != nil { - return types.Capability{}, []byte{}, err - } - return capa, data, nil -} - -// ListCaps list all capabilities of registry -func (l LocalRegistry) ListCaps() ([]types.Capability, error) { - glob := filepath.Join(filepath.Clean(l.absPath), "*") - files, _ := filepath.Glob(glob) - capas := make([]types.Capability, 0) - for _, file := range files { - // nolint:gosec - data, err := os.ReadFile(file) - if err != nil { - return nil, err - } - capa, err := RegistryFile{ - data: data, - name: path.Base(file), - }.toAddon() - if err != nil { - fmt.Printf("parsing file: %s err: %s\n", file, err) - continue - } - capas = append(capas, capa) - } - return capas, nil -} -func (item RegistryFile) toAddon() (types.Capability, error) { - dm, err := (&common.Args{}).GetDiscoveryMapper() - if err != nil { - return types.Capability{}, err - } - capability, err := ParseCapability(dm, item.data) - if err != nil { - return types.Capability{}, err - } - return capability, nil -} - -// RegistryFile describes a file item in registry -type RegistryFile struct { - data []byte // file content - name string // file's name -} - -// ListBucketResult describe a file list from OSS -type ListBucketResult struct { - File []string `xml:"Contents>Key"` - Count int `xml:"KeyCount"` -}