Feat: sync application from CR to data store (#3428)

* Feat: sync application from CR to data store

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

* Feature: address comments

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

* Feat: add migrate database feature to avoid max 63 charactor in kubeapi storage

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

* update the sync data

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

pkg/apiserver/model/application.go

@@ -18,13 +18,14 @@ package model
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
 )
 
 func init() {
-	RegistModel(&ApplicationComponent{}, &ApplicationPolicy{}, &Application{}, &ApplicationRevision{}, &ApplicationTrigger{})
+	RegisterModel(&ApplicationComponent{}, &ApplicationPolicy{}, &Application{}, &ApplicationRevision{}, &ApplicationTrigger{})
 }
 
 // Application application delivery model
@@ -43,7 +44,15 @@ func (a *Application) TableName() string {
 	return tableNamePrefix + "application"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (a *Application) ShortTableName() string {
+	return "app"
+}
+
 // PrimaryKey return custom primary key
+// the app primary key is the app name, so the app name is globally unique in every namespace
+// when the app is synced from CR, the first synced one be same with app name,
+// if there's any conflicts, the name will be composed by <appname>-<namespace>
 func (a *Application) PrimaryKey() string {
 	return a.Name
 }
@@ -60,6 +69,18 @@ func (a *Application) Index() map[string]string {
 	return index
 }
 
+// GetAppNameForSynced will trim namespace suffix for synced CR
+func (a *Application) GetAppNameForSynced() string {
+	if a.Labels == nil {
+		return a.Name
+	}
+	namespace := a.Labels[LabelSyncNamespace]
+	if namespace == "" {
+		return a.Name
+	}
+	return strings.TrimSuffix(a.Name, "-"+namespace)
+}
+
 // ClusterSelector cluster selector
 type ClusterSelector struct {
 	Name string `json:"name"`
@@ -102,6 +123,11 @@ func (a *ApplicationComponent) TableName() string {
 	return tableNamePrefix + "application_component"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (a *ApplicationComponent) ShortTableName() string {
+	return "app_cmp"
+}
+
 // PrimaryKey return custom primary key
 func (a *ApplicationComponent) PrimaryKey() string {
 	return fmt.Sprintf("%s-%s", a.AppPrimaryKey, a.Name)
@@ -138,6 +164,11 @@ func (a *ApplicationPolicy) TableName() string {
 	return tableNamePrefix + "application_policy"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (a *ApplicationPolicy) ShortTableName() string {
+	return "app_plc"
+}
+
 // PrimaryKey return custom primary key
 func (a *ApplicationPolicy) PrimaryKey() string {
 	return fmt.Sprintf("%s-%s", a.AppPrimaryKey, a.Name)
@@ -270,6 +301,11 @@ func (a *ApplicationRevision) TableName() string {
 	return tableNamePrefix + "application_revision"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (a *ApplicationRevision) ShortTableName() string {
+	return "app_rev"
+}
+
 // PrimaryKey return custom primary key
 func (a *ApplicationRevision) PrimaryKey() string {
 	return fmt.Sprintf("%s-%s", a.AppPrimaryKey, a.Version)
@@ -349,6 +385,11 @@ func (w *ApplicationTrigger) TableName() string {
 	return tableNamePrefix + "trigger"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (w *ApplicationTrigger) ShortTableName() string {
+	return "app_tg"
+}
+
 // PrimaryKey return custom primary key
 func (w *ApplicationTrigger) PrimaryKey() string {
 	return w.Token

pkg/apiserver/model/cluster.go

@@ -23,7 +23,7 @@ import (
 )
 
 func init() {
-	RegistModel(&Cluster{})
+	RegisterModel(&Cluster{})
 }
 
 // ProviderInfo describes the information from provider API
@@ -82,6 +82,11 @@ func (c *Cluster) TableName() string {
 	return tableNamePrefix + "cluster"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (c *Cluster) ShortTableName() string {
+	return "cls"
+}
+
 // PrimaryKey primary key for datastore
 func (c *Cluster) PrimaryKey() string {
 	return c.Name

pkg/apiserver/model/env.go

@@ -17,7 +17,7 @@ limitations under the License.
 package model
 
 func init() {
-	RegistModel(&Env{})
+	RegisterModel(&Env{})
 }
 
 // Env models the data of env in database
@@ -42,6 +42,11 @@ func (p *Env) TableName() string {
 	return tableNamePrefix + "env"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (p *Env) ShortTableName() string {
+	return "ev"
+}
+
 // PrimaryKey return custom primary key
 func (p *Env) PrimaryKey() string {
 	return p.Name

pkg/apiserver/model/envbinding.go

@@ -19,13 +19,14 @@ package model
 import "fmt"
 
 func init() {
-	RegistModel(&EnvBinding{})
+	RegisterModel(&EnvBinding{})
 }
 
 // EnvBinding application env binding
 type EnvBinding struct {
 	BaseModel
 	AppPrimaryKey   string           `json:"appPrimaryKey"`
+	AppDeployName   string           `json:"appDeployName"`
 	Name            string           `json:"name"`
 	ComponentsPatch []ComponentPatch `json:"componentsPatchs"`
 }
@@ -50,6 +51,11 @@ func (e *EnvBinding) TableName() string {
 	return tableNamePrefix + "envbinding"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (e *EnvBinding) ShortTableName() string {
+	return "evb"
+}
+
 // PrimaryKey return custom primary key
 func (e *EnvBinding) PrimaryKey() string {
 	return fmt.Sprintf("%s-%s", e.AppPrimaryKey, e.Name)

pkg/apiserver/model/model.go

@@ -30,28 +30,37 @@ import (
 
 var tableNamePrefix = "vela_"
 
-var registedModels = map[string]Interface{}
+var registeredModels = map[string]Interface{}
 
 // Interface model interface
 type Interface interface {
 	TableName() string
+	ShortTableName() string
 }
 
-// RegistModel regist model
-func RegistModel(models ...Interface) {
+// RegisterModel register model
+func RegisterModel(models ...Interface) {
 	for _, model := range models {
-		if _, exist := registedModels[model.TableName()]; exist {
+		if _, exist := registeredModels[model.TableName()]; exist {
 			panic(fmt.Errorf("model table name %s conflict", model.TableName()))
 		}
-		registedModels[model.TableName()] = model
+		registeredModels[model.TableName()] = model
 	}
 }
 
+// GetRegisterModels will return the register models
+func GetRegisterModels() map[string]Interface {
+	return registeredModels
+}
+
 // JSONStruct json struct, same with runtime.RawExtension
 type JSONStruct map[string]interface{}
 
-// NewJSONStruct new jsonstruct from runtime.RawExtension
+// NewJSONStruct new json struct from runtime.RawExtension
 func NewJSONStruct(raw *runtime.RawExtension) (*JSONStruct, error) {
+	if raw == nil || raw.Raw == nil {
+		return nil, nil
+	}
 	var data JSONStruct
 	err := json.Unmarshal(raw.Raw, &data)
 	if err != nil {
@@ -60,7 +69,7 @@ func NewJSONStruct(raw *runtime.RawExtension) (*JSONStruct, error) {
 	return &data, nil
 }
 
-// NewJSONStructByString new jsonstruct from string
+// NewJSONStructByString new json struct from string
 func NewJSONStructByString(source string) (*JSONStruct, error) {
 	if source == "" {
 		return nil, nil
@@ -73,7 +82,7 @@ func NewJSONStructByString(source string) (*JSONStruct, error) {
 	return &data, nil
 }
 
-// NewJSONStructByStruct new jsonstruct from strcut object
+// NewJSONStructByStruct new json struct from struct object
 func NewJSONStructByStruct(object interface{}) (*JSONStruct, error) {
 	if object == nil {
 		return nil, nil

pkg/apiserver/model/project.go

@@ -17,10 +17,10 @@ limitations under the License.
 package model
 
 func init() {
-	RegistModel(&Project{})
+	RegisterModel(&Project{})
 }
 
-// Project project model
+// Project basic model
 type Project struct {
 	BaseModel
 	Name        string `json:"name"`
@@ -33,6 +33,11 @@ func (p *Project) TableName() string {
 	return tableNamePrefix + "project"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (p *Project) ShortTableName() string {
+	return "pj"
+}
+
 // PrimaryKey return custom primary key
 func (p *Project) PrimaryKey() string {
 	return p.Name

pkg/apiserver/model/system_info.go

@@ -17,7 +17,7 @@ limitations under the License.
 package model
 
 func init() {
-	RegistModel(&SystemInfo{})
+	RegisterModel(&SystemInfo{})
 }
 
 // LoginType is the type of login
@@ -43,6 +43,11 @@ func (u *SystemInfo) TableName() string {
 	return tableNamePrefix + "system_info"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (u *SystemInfo) ShortTableName() string {
+	return "sysi"
+}
+
 // PrimaryKey return custom primary key
 func (u *SystemInfo) PrimaryKey() string {
 	return u.InstallID

pkg/apiserver/model/target.go

@@ -17,7 +17,7 @@ limitations under the License.
 package model
 
 func init() {
-	RegistModel(&Target{})
+	RegisterModel(&Target{})
 }
 
 // Target defines the delivery target information for the application
@@ -36,6 +36,11 @@ func (d *Target) TableName() string {
 	return tableNamePrefix + "target"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (d *Target) ShortTableName() string {
+	return "tg"
+}
+
 // PrimaryKey return custom primary key
 func (d *Target) PrimaryKey() string {
 	return d.Name

pkg/apiserver/model/user.go

@@ -19,7 +19,7 @@ package model
 import "strings"
 
 func init() {
-	RegistModel(&User{})
+	RegisterModel(&User{})
 }
 
 // User is the model of user
@@ -37,6 +37,11 @@ func (u *User) TableName() string {
 	return tableNamePrefix + "user"
 }
 
+// ShortTableName return custom table name
+func (u *User) ShortTableName() string {
+	return "usr"
+}
+
 // PrimaryKey return custom primary key
 func (u *User) PrimaryKey() string {
 	return verifyUserValue(u.Name)

pkg/apiserver/model/whole.go

@@ -0,0 +1,73 @@
+/*
+ Copyright 2022 The KubeVela Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ 	http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package model
+
+const (
+	// AutoGenDesc describes the metadata in datastore that's automatically generated
+	AutoGenDesc = "Automatically converted from KubeVela Application in Kubernetes."
+
+	// AutoGenProj describes the automatically created project
+	AutoGenProj = "Automatically generated by sync mechanism."
+
+	// AutoGenEnvNamePrefix describes the common prefix for auto-generated env
+	AutoGenEnvNamePrefix = "synced-"
+	// AutoGenComp describes the creator of component that is auto-generated
+	AutoGenComp = "synced-comp"
+	// AutoGenPolicy describes the creator of policy that is auto-generated
+	AutoGenPolicy = "synced-policy"
+	// AutoGenRefPolicy describes the creator of policy that is auto-generated, this differs from AutoGenPolicy as the policy is referenced ones
+	AutoGenRefPolicy = "synced-ref-policy"
+	// AutoGenWorkflowNamePrefix describes the common prefix for auto-generated workflow
+	AutoGenWorkflowNamePrefix = "synced-"
+	// AutoGenTargetNamePrefix describes the common prefix for auto-generated target
+	AutoGenTargetNamePrefix = "synced-"
+
+	// LabelSyncGeneration describes the generation synced from
+	LabelSyncGeneration = "ux.oam.dev/synced-generation"
+	// LabelSyncNamespace describes the namespace synced from
+	LabelSyncNamespace = "ux.oam.dev/from-namespace"
+)
+
+// DataStoreApp is a memory struct that describes the model of an application in datastore
+type DataStoreApp struct {
+	AppMeta  *Application
+	Env      *Env
+	Eb       *EnvBinding
+	Comps    []*ApplicationComponent
+	Policies []*ApplicationPolicy
+	Workflow *Workflow
+	Targets  []*Target
+}
+
+const (
+
+	// DefaultInitName is default object name for initialization
+	DefaultInitName = "default"
+
+	// DefaultAddonProject is default addon projects
+	DefaultAddonProject = "addons"
+
+	// DefaultInitNamespace is default namespace name for initialization
+	DefaultInitNamespace = "default"
+
+	// DefaultTargetDescription describes default target created
+	DefaultTargetDescription = "Default target is created by velaux system automatically."
+	// DefaultEnvDescription describes default env created
+	DefaultEnvDescription = "Default environment is created by velaux system automatically."
+	// DefaultProjectDescription describes the default project created
+	DefaultProjectDescription = "Default project is created by velaux system automatically."
+)

pkg/apiserver/model/workflow.go

@@ -25,8 +25,8 @@ import (
 )
 
 func init() {
-	RegistModel(&Workflow{})
-	RegistModel(&WorkflowRecord{})
+	RegisterModel(&Workflow{})
+	RegisterModel(&WorkflowRecord{})
 }
 
 // Workflow application delivery database model
@@ -61,6 +61,11 @@ func (w *Workflow) TableName() string {
 	return tableNamePrefix + "workflow"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (w *Workflow) ShortTableName() string {
+	return "wf"
+}
+
 // PrimaryKey return custom primary key
 func (w *Workflow) PrimaryKey() string {
 	return fmt.Sprintf("%s-%s", w.AppPrimaryKey, w.Name)
@@ -118,6 +123,11 @@ func (w *WorkflowRecord) TableName() string {
 	return tableNamePrefix + "workflow_record"
 }
 
+// ShortTableName is the compressed version of table name for kubeapi storage and others
+func (w *WorkflowRecord) ShortTableName() string {
+	return "wfr"
+}
+
 // PrimaryKey return custom primary key
 func (w *WorkflowRecord) PrimaryKey() string {
 	return w.Name