From 19a542ff1168d30b9f5559290df21e8a0210e142 Mon Sep 17 00:00:00 2001 From: Jianbo Sun Date: Tue, 11 Jan 2022 17:28:45 +0800 Subject: [PATCH] Feat: support vela up from remote url file (#3075) Signed-off-by: Jianbo Sun --- references/appfile/api/appfile.go | 61 ++++---------- references/appfile/api/appfile_test.go | 30 ++----- references/cli/up.go | 32 ++++--- references/cli/up_test.go | 11 +-- references/common/application.go | 110 +++++++++++++++---------- references/common/common.go | 4 +- 6 files changed, 105 insertions(+), 143 deletions(-) diff --git a/references/appfile/api/appfile.go b/references/appfile/api/appfile.go index cf6ece3c4..565925c23 100644 --- a/references/appfile/api/appfile.go +++ b/references/appfile/api/appfile.go @@ -24,15 +24,11 @@ import ( "strings" "time" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" "github.com/oam-dev/kubevela/apis/core.oam.dev/common" - "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/pkg/builtin" - "github.com/oam-dev/kubevela/pkg/oam" cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" "github.com/oam-dev/kubevela/references/appfile/template" ) @@ -103,20 +99,17 @@ func LoadFromFile(filename string) (*AppFile, error) { if err != nil { return nil, err } + return LoadFromBytes(b) +} + +// LoadFromBytes will load AppFile from bytes +func LoadFromBytes(b []byte) (*AppFile, error) { af := NewAppFile() - // Add JSON format appfile support - ext := filepath.Ext(filename) - switch ext { - case ".yaml", ".yml": - err = yaml.Unmarshal(b, af) - case ".json": + var err error + if json.Valid(b) { af, err = JSONToYaml(b, af) - default: - if json.Valid(b) { - af, err = JSONToYaml(b, af) - } else { - err = yaml.Unmarshal(b, af) - } + } else { + err = yaml.Unmarshal(b, af) } if err != nil { return nil, err @@ -140,16 +133,15 @@ func (app *AppFile) ExecuteAppfileTasks(io cmdutil.IOStreams) error { return nil } -// BuildOAMApplication renders Appfile into Application, Scopes and other K8s Resources. -func (app *AppFile) BuildOAMApplication(namespace string, io cmdutil.IOStreams, tm template.Manager, silence bool) (*v1beta1.Application, []oam.Object, error) { +// ConvertToApplication renders Appfile into Application, Scopes and other K8s Resources. +func (app *AppFile) ConvertToApplication(namespace string, io cmdutil.IOStreams, tm template.Manager, silence bool) (*v1beta1.Application, error) { if err := app.ExecuteAppfileTasks(io); err != nil { if strings.Contains(err.Error(), "'image' : not found") { - return nil, nil, ErrImageNotDefined + return nil, ErrImageNotDefined } - return nil, nil, err + return nil, err } // auxiliaryObjects currently include OAM Scope Custom Resources and ConfigMaps - var auxiliaryObjects []oam.Object servApp := new(v1beta1.Application) servApp.SetNamespace(namespace) servApp.SetName(app.Name) @@ -160,33 +152,10 @@ func (app *AppFile) BuildOAMApplication(namespace string, io cmdutil.IOStreams, for serviceName, svc := range app.GetServices() { comp, err := svc.RenderServiceToApplicationComponent(tm, serviceName) if err != nil { - return nil, nil, err + return nil, err } servApp.Spec.Components = append(servApp.Spec.Components, comp) } servApp.SetGroupVersionKind(v1beta1.SchemeGroupVersion.WithKind("Application")) - auxiliaryObjects = append(auxiliaryObjects, addDefaultHealthScopeToApplication(servApp)) - return servApp, auxiliaryObjects, nil -} - -func addDefaultHealthScopeToApplication(app *v1beta1.Application) *v1alpha2.HealthScope { - health := &v1alpha2.HealthScope{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1alpha2.HealthScopeGroupVersionKind.GroupVersion().String(), - Kind: v1alpha2.HealthScopeKind, - }, - } - health.Name = FormatDefaultHealthScopeName(app.Name) - health.Namespace = app.Namespace - health.Spec.WorkloadReferences = make([]corev1.ObjectReference, 0) - for i := range app.Spec.Components { - // FIXME(wonderflow): the hardcode health scope should be fixed. - app.Spec.Components[i].Scopes = map[string]string{DefaultHealthScopeKey: health.Name} - } - return health -} - -// FormatDefaultHealthScopeName will create a default health scope name. -func FormatDefaultHealthScopeName(appName string) string { - return appName + "-default-health" + return servApp, nil } diff --git a/references/appfile/api/appfile_test.go b/references/appfile/api/appfile_test.go index b519a54d0..d1e647a79 100644 --- a/references/appfile/api/appfile_test.go +++ b/references/appfile/api/appfile_test.go @@ -21,13 +21,11 @@ import ( "testing" "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" "github.com/oam-dev/kubevela/apis/core.oam.dev/common" - "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/oam" @@ -80,7 +78,6 @@ func TestBuildOAMApplication2(t *testing.T) { Properties: &runtime.RawExtension{ Raw: []byte("{\"image\":\"busybox\"}"), }, - Scopes: map[string]string{"healthscopes.core.oam.dev": "test-default-health"}, }, }, }, @@ -114,7 +111,6 @@ func TestBuildOAMApplication2(t *testing.T) { Properties: &runtime.RawExtension{ Raw: []byte("{\"image\":\"busybox\"}"), }, - Scopes: map[string]string{"healthscopes.core.oam.dev": "test-default-health"}, Traits: []common.ApplicationTrait{ { Type: "scaler", @@ -132,7 +128,7 @@ func TestBuildOAMApplication2(t *testing.T) { for _, tcase := range testCases { tcase.expectApp.Namespace = expectNs - o, _, err := tcase.appFile.BuildOAMApplication(expectNs, cmdutil.IOStreams{ + o, err := tcase.appFile.ConvertToApplication(expectNs, cmdutil.IOStreams{ In: os.Stdin, Out: os.Stdout, }, tm, false) @@ -258,9 +254,8 @@ outputs: ingress: { }, Spec: v1beta1.ApplicationSpec{ Components: []common.ApplicationComponent{{ - Type: "webservice", - Name: "express-server", - Scopes: map[string]string{"healthscopes.core.oam.dev": "myapp-default-health"}, + Type: "webservice", + Name: "express-server", Properties: &runtime.RawExtension{ Raw: []byte(`{"image": "oamdev/testapp:v1", "cmd": ["node", "server.js"]}`), }, @@ -282,7 +277,6 @@ outputs: ingress: { Raw: []byte(`{"image":"bitnami/mongodb:3.6.20","cmd": ["mongodb"]}`), }, Traits: []common.ApplicationTrait{}, - Scopes: map[string]string{"healthscopes.core.oam.dev": "myapp-default-health"}, }) ac3 := ac1.DeepCopy() @@ -291,15 +285,6 @@ outputs: ingress: { // TODO application 那边补测试: // 2. 1对多的情况，多对1 的情况 - health := &v1alpha2.HealthScope{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1alpha2.HealthScopeGroupVersionKind.GroupVersion().String(), - Kind: v1alpha2.HealthScopeKind, - }, - } - health.Name = FormatDefaultHealthScopeName("myapp") - health.Namespace = "default" - health.Spec.WorkloadReferences = make([]corev1.ObjectReference, 0) type args struct { appfileData string workloadTemplates map[string]string @@ -326,7 +311,7 @@ outputs: ingress: { }, want: want{ app: ac1, - objs: []oam.Object{health}, + objs: []oam.Object{}, }, }, "two services should generate two components and one appconfig": { @@ -342,7 +327,7 @@ outputs: ingress: { }, want: want{ app: ac2, - objs: []oam.Object{health}, + objs: []oam.Object{}, }, }, "no image should fail": { @@ -378,7 +363,7 @@ outputs: ingress: { } } - application, objects, err := app.BuildOAMApplication("default", io, tm, false) + application, err := app.ConvertToApplication("default", io, tm, false) if c.want.err != nil { assert.Equal(t, c.want.err, err) return @@ -412,9 +397,6 @@ outputs: ingress: { } assert.Equal(t, true, found, "no component found for %s", comp.Name) } - for idx, v := range objects { - assert.Equal(t, c.want.objs[idx], v) - } }) } diff --git a/references/cli/up.go b/references/cli/up.go index 6adc4f5aa..1b2ceee06 100644 --- a/references/cli/up.go +++ b/references/cli/up.go @@ -17,9 +17,6 @@ limitations under the License. package cli import ( - "os" - "path/filepath" - "github.com/pkg/errors" "github.com/spf13/cobra" "sigs.k8s.io/yaml" @@ -38,7 +35,7 @@ func NewUpCommand(c common2.Args, order string, ioStream cmdutil.IOStreams) *cob Use: "up", DisableFlagsInUseLine: true, Short: "Apply an appfile or application from file", - Long: "Create or update vela application from file, both appfile or application object format are supported.", + Long: "Create or update vela application from file or URL, both appfile or application object format are supported.", Annotations: map[string]string{ types.TagCommandOrder: order, types.TagCommandType: types.TypeStart, @@ -52,21 +49,13 @@ func NewUpCommand(c common2.Args, order string, ioStream cmdutil.IOStreams) *cob if err != nil { return err } - fileContent, err := os.ReadFile(filepath.Clean(*appFilePath)) + + body, err := common.ReadRemoteOrLocalPath(*appFilePath) if err != nil { return err } - var app corev1beta1.Application - err = yaml.Unmarshal(fileContent, &app) - if err != nil { - return errors.Wrap(err, "File format is illegal, only support vela appfile format or OAM Application object yaml") - } - if app.APIVersion != "" && app.Kind != "" { - err = common.ApplyApplication(app, ioStream, kubecli) - if err != nil { - return err - } - } else { + + if common.IsAppfile(body) { o := &common.AppfileOptions{ Kubecli: kubecli, IO: ioStream, @@ -74,11 +63,20 @@ func NewUpCommand(c common2.Args, order string, ioStream cmdutil.IOStreams) *cob } return o.Run(*appFilePath, o.Namespace, c) } + var app corev1beta1.Application + err = yaml.Unmarshal(body, &app) + if err != nil { + return errors.Wrap(err, "File format is illegal, only support vela appfile format or OAM Application object yaml") + } + err = common.ApplyApplication(app, ioStream, kubecli) + if err != nil { + return err + } return nil }, } cmd.SetOut(ioStream.Out) - cmd.Flags().StringVarP(appFilePath, "file", "f", "", "specify file path for appfile or application") + cmd.Flags().StringVarP(appFilePath, "file", "f", "", "specify file path for appfile or application, it could be a remote url.") addNamespaceAndEnvArg(cmd) return cmd diff --git a/references/cli/up_test.go b/references/cli/up_test.go index 14694b8b3..405326c9d 100644 --- a/references/cli/up_test.go +++ b/references/cli/up_test.go @@ -18,26 +18,19 @@ package cli import ( "fmt" - "os" "testing" "github.com/stretchr/testify/assert" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" - "github.com/oam-dev/kubevela/pkg/utils/util" "github.com/oam-dev/kubevela/references/common" ) func TestUp(t *testing.T) { - ioStream := util.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} - namespace := "up-ns" - o := common.AppfileOptions{ - IO: ioStream, - Namespace: namespace, - } + app := &v1beta1.Application{} app.Name = "app-up" - msg := o.Info(app) + msg := common.Info(app) assert.Contains(t, msg, "App has been deployed") assert.Contains(t, msg, fmt.Sprintf("App status: vela status %s", app.Name)) } diff --git a/references/common/application.go b/references/common/application.go index 90c843956..2b83261ed 100644 --- a/references/common/application.go +++ b/references/common/application.go @@ -26,9 +26,8 @@ import ( "strings" "time" - "github.com/fatih/color" - "github.com/crossplane/crossplane-runtime/pkg/meta" + "github.com/fatih/color" "github.com/gosuri/uilive" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -240,33 +239,71 @@ func (o *DeleteOptions) DeleteComponent(io cmdutil.IOStreams) error { return nil } -func saveAndLoadRemoteAppfile(url string) (*api.AppFile, error) { - body, err := common.HTTPGet(context.Background(), url) +// LoadAppFile will load vela appfile from remote URL or local file system. +func LoadAppFile(pathOrURL string) (*api.AppFile, error) { + body, err := ReadRemoteOrLocalPath(pathOrURL) if err != nil { return nil, err } - af := api.NewAppFile() - ext := filepath.Ext(url) - dest := "Appfile" - switch ext { - case ".json": - dest = "vela.json" - af, err = api.JSONToYaml(body, af) - case ".yaml", ".yml": - dest = "vela.yaml" - err = yaml.Unmarshal(body, af) - default: - if j.Valid(body) { - af, err = api.JSONToYaml(body, af) - } else { - err = yaml.Unmarshal(body, af) + return api.LoadFromBytes(body) +} + +// ReadRemoteOrLocalPath will read a path remote or locally +func ReadRemoteOrLocalPath(pathOrURL string) ([]byte, error) { + var body []byte + var err error + if strings.HasPrefix(pathOrURL, "https://") || strings.HasPrefix(pathOrURL, "http://") { + body, err = common.HTTPGet(context.Background(), pathOrURL) + if err != nil { + return nil, err + } + if err = localSave(pathOrURL, body); err != nil { + return nil, err + } + } else { + body, err = os.ReadFile(filepath.Clean(pathOrURL)) + if err != nil { + return nil, err } } + return body, nil +} + +// IsAppfile check if a file is Appfile format or application format, return true if it's appfile, false means application object +func IsAppfile(body []byte) bool { + if j.Valid(body) { + // we only support json format for appfile + return true + } + res := map[string]interface{}{} + err := yaml.Unmarshal(body, &res) if err != nil { - return nil, err + return false + } + // appfile didn't have apiVersion + if _, ok := res["apiVersion"]; ok { + return false + } + return true +} + +func localSave(url string, body []byte) error { + var name string + ext := filepath.Ext(url) + switch ext { + case ".json": + name = "vela.json" + case ".yaml", ".yml": + name = "vela.yaml" + default: + if j.Valid(body) { + name = "vela.json" + } else { + name = "vela.yaml" + } } //nolint:gosec - return af, os.WriteFile(dest, body, 0644) + return os.WriteFile(name, body, 0644) } // ExportFromAppFile exports Application from appfile object @@ -279,7 +316,7 @@ func (o *AppfileOptions) ExportFromAppFile(app *api.AppFile, namespace string, q appHandler := appfile.NewApplication(app, tm) // new - retApplication, scopes, err := appHandler.BuildOAMApplication(o.Namespace, o.IO, appHandler.Tm, quiet) + retApplication, err := appHandler.ConvertToApplication(o.Namespace, o.IO, appHandler.Tm, quiet) if err != nil { return nil, nil, err } @@ -294,19 +331,9 @@ func (o *AppfileOptions) ExportFromAppFile(app *api.AppFile, namespace string, q } w.WriteByte('\n') - for _, scope := range scopes { - w.WriteString("---\n") - err = enc.Encode(scope, &w) - if err != nil { - return nil, nil, fmt.Errorf("yaml encode scope (%s) failed: %w", scope.GetName(), err) - } - w.WriteByte('\n') - } - result := &BuildResult{ appFile: app, application: retApplication, - scopes: scopes, } return result, w.Bytes(), nil } @@ -316,14 +343,10 @@ func (o *AppfileOptions) Export(filePath, namespace string, quiet bool, c common var app *api.AppFile var err error if !quiet { - o.IO.Info("Parsing vela appfile ...") + o.IO.Info("Parsing vela application file ...") } if filePath != "" { - if strings.HasPrefix(filePath, "https://") || strings.HasPrefix(filePath, "http://") { - app, err = saveAndLoadRemoteAppfile(filePath) - } else { - app, err = api.LoadFromFile(filePath) - } + app, err = LoadAppFile(filePath) } else { app, err = api.Load() } @@ -383,7 +406,7 @@ func (o *AppfileOptions) ApplyApp(app *corev1beta1.Application, scopes []oam.Obj if err := o.apply(app, scopes); err != nil { return err } - o.IO.Infof(o.Info(app)) + o.IO.Infof(Info(app)) return nil } @@ -395,7 +418,7 @@ func (o *AppfileOptions) apply(app *corev1beta1.Application, scopes []oam.Object } // Info shows the status of each service in the Appfile -func (o *AppfileOptions) Info(app *corev1beta1.Application) string { +func Info(app *corev1beta1.Application) string { appName := app.Name var appUpMessage = "✅ App has been deployed 🚀🚀🚀\n" + fmt.Sprintf(" Port forward: vela port-forward %s\n", appName) + @@ -413,7 +436,7 @@ func ApplyApplication(app corev1beta1.Application, ioStream cmdutil.IOStreams, c if app.Namespace == "" { app.Namespace = types.DefaultAppNamespace } - _, err := ioStream.Out.Write([]byte("Applying an application in K8S format...\n")) + _, err := ioStream.Out.Write([]byte("Applying an application in vela K8s object format...\n")) if err != nil { return err } @@ -422,9 +445,6 @@ func ApplyApplication(app corev1beta1.Application, ioStream cmdutil.IOStreams, c if err != nil { return err } - _, err = ioStream.Out.Write([]byte("Successfully apply application")) - if err != nil { - return err - } + ioStream.Infof(Info(&app)) return nil } diff --git a/references/common/common.go b/references/common/common.go index 1fbfdcaa3..76029ce1f 100644 --- a/references/common/common.go +++ b/references/common/common.go @@ -28,10 +28,10 @@ import ( // BuildRun will build application and deploy from Appfile func BuildRun(ctx context.Context, app *api.Application, client client.Client, namespace string, io util.IOStreams) error { - o, scopes, err := app.BuildOAMApplication(namespace, io, app.Tm, true) + o, err := app.ConvertToApplication(namespace, io, app.Tm, true) if err != nil { return err } - return appfile.Run(ctx, client, o, scopes) + return appfile.Run(ctx, client, o, nil) }