From d9a676a6882687a4bf0a816fe5ec0079bea26907 Mon Sep 17 00:00:00 2001 From: Jianbo Sun Date: Mon, 21 Mar 2022 19:28:28 +0800 Subject: [PATCH] Feat: add page index for env list API (#3482) * Feat: add page index for env list API Signed-off-by: Jianbo Sun * Fix: add check for target create without namespace or name Signed-off-by: Jianbo Sun * Fix: adress commnets Signed-off-by: Jianbo Sun --- docs/apidoc/swagger.json | 86 +++++++++++-------- pkg/apiserver/rest/apis/v1/types.go | 4 +- pkg/apiserver/rest/usecase/env.go | 14 +-- pkg/apiserver/rest/usecase/project.go | 20 +++-- pkg/apiserver/rest/usecase/project_test.go | 8 +- pkg/apiserver/rest/usecase/target_model.go | 3 + pkg/apiserver/rest/utils/bcode/008_target.go | 3 + pkg/apiserver/rest/webservice/env.go | 2 +- pkg/apiserver/rest/webservice/project.go | 12 ++- .../rest/webservice/webservice_suite_test.go | 74 +++++++++++++++- 10 files changed, 165 insertions(+), 61 deletions(-) diff --git a/docs/apidoc/swagger.json b/docs/apidoc/swagger.json index 199f8fb5b..568c97151 100644 --- a/docs/apidoc/swagger.json +++ b/docs/apidoc/swagger.json @@ -6359,12 +6359,12 @@ }, "v1.ApplicationDeployResponse": { "required": [ - "createTime", - "note", - "version", "status", + "createTime", + "version", "envName", - "triggerType" + "triggerType", + "note" ], "properties": { "codeInfo": { @@ -7420,10 +7420,10 @@ }, "v1.DetailAddonResponse": { "required": [ - "invisible", "name", - "description", + "invisible", "version", + "description", "icon", "schema", "uiSchema", @@ -7497,11 +7497,11 @@ }, "v1.DetailApplicationResponse": { "required": [ - "icon", - "name", "alias", "project", "description", + "icon", + "name", "createTime", "updateTime", "policies", @@ -7568,20 +7568,20 @@ }, "v1.DetailClusterResponse": { "required": [ - "name", - "alias", - "labels", - "dashboardURL", - "kubeConfigSecret", "createTime", - "updateTime", - "reason", - "icon", - "provider", "description", "status", "apiServerURL", + "updateTime", + "alias", + "reason", + "name", + "icon", + "labels", + "provider", + "dashboardURL", "kubeConfig", + "kubeConfigSecret", "resourceInfo" ], "properties": { @@ -7639,14 +7639,14 @@ }, "v1.DetailComponentResponse": { "required": [ - "updateTime", "creator", "name", "alias", "type", - "main", "createTime", "appPrimaryKey", + "main", + "updateTime", "definition" ], "properties": { @@ -7748,13 +7748,13 @@ }, "v1.DetailPolicyResponse": { "required": [ + "name", "type", "description", "creator", "properties", "createTime", - "updateTime", - "name" + "updateTime" ], "properties": { "createTime": { @@ -7784,16 +7784,16 @@ }, "v1.DetailRevisionResponse": { "required": [ - "note", - "workflowName", - "createTime", - "triggerType", - "updateTime", "appPrimaryKey", - "deployUser", - "envName", "version", "status", + "workflowName", + "updateTime", + "deployUser", + "note", + "triggerType", + "createTime", + "envName", "reason" ], "properties": { @@ -7848,8 +7848,8 @@ }, "v1.DetailTargetResponse": { "required": [ - "updateTime", "createTime", + "updateTime", "name" ], "properties": { @@ -7887,11 +7887,11 @@ }, "v1.DetailUserResponse": { "required": [ + "email", "disabled", "createTime", "lastLoginTime", "name", - "email", "projects" ], "properties": { @@ -7925,12 +7925,12 @@ }, "v1.DetailWorkflowRecordResponse": { "required": [ - "status", - "name", "namespace", "workflowName", "workflowAlias", "applicationRevision", + "status", + "name", "deployTime", "deployUser", "note", @@ -7982,14 +7982,14 @@ }, "v1.DetailWorkflowResponse": { "required": [ - "description", "default", "envName", - "createTime", + "updateTime", "name", "alias", + "description", "enable", - "updateTime" + "createTime" ], "properties": { "alias": { @@ -8348,7 +8348,8 @@ }, "v1.ListEnvResponse": { "required": [ - "envs" + "envs", + "total" ], "properties": { "envs": { @@ -8356,6 +8357,10 @@ "items": { "$ref": "#/definitions/v1.Env" } + }, + "total": { + "type": "integer", + "format": "int64" } } }, @@ -8374,7 +8379,8 @@ }, "v1.ListProjectResponse": { "required": [ - "projects" + "projects", + "total" ], "properties": { "projects": { @@ -8382,6 +8388,10 @@ "items": { "$ref": "#/definitions/v1.ProjectBase" } + }, + "total": { + "type": "integer", + "format": "int64" } } }, @@ -8662,11 +8672,11 @@ }, "v1.SystemInfoResponse": { "required": [ + "createTime", "updateTime", "installID", "enableCollection", "loginType", - "createTime", "systemVersion" ], "properties": { diff --git a/pkg/apiserver/rest/apis/v1/types.go b/pkg/apiserver/rest/apis/v1/types.go index 04986baaa..f539a2db3 100644 --- a/pkg/apiserver/rest/apis/v1/types.go +++ b/pkg/apiserver/rest/apis/v1/types.go @@ -668,6 +668,7 @@ type ApplicationTemplateVersion struct { // ListProjectResponse list project response body type ListProjectResponse struct { Projects []*ProjectBase `json:"projects"` + Total int64 `json:"total"` } // ProjectBase project base model @@ -712,7 +713,8 @@ type ListEnvOptions struct { // ListEnvResponse response the while env list type ListEnvResponse struct { - Envs []*Env `json:"envs"` + Envs []*Env `json:"envs"` + Total int64 `json:"total"` } // CreateEnvRequest contains the env data as request body diff --git a/pkg/apiserver/rest/usecase/env.go b/pkg/apiserver/rest/usecase/env.go index 3b44e5102..43e1854cd 100644 --- a/pkg/apiserver/rest/usecase/env.go +++ b/pkg/apiserver/rest/usecase/env.go @@ -38,7 +38,7 @@ import ( // EnvUsecase defines the API of Env. type EnvUsecase interface { GetEnv(ctx context.Context, envName string) (*model.Env, error) - ListEnvs(ctx context.Context, page, pageSize int, listOption apisv1.ListEnvOptions) ([]*apisv1.Env, error) + ListEnvs(ctx context.Context, page, pageSize int, listOption apisv1.ListEnvOptions) (*apisv1.ListEnvResponse, error) DeleteEnv(ctx context.Context, envName string) error CreateEnv(ctx context.Context, req apisv1.CreateEnvRequest) (*apisv1.Env, error) UpdateEnv(ctx context.Context, envName string, req apisv1.UpdateEnvRequest) (*apisv1.Env, error) @@ -95,7 +95,7 @@ func (p *envUsecaseImpl) DeleteEnv(ctx context.Context, envName string) error { } // ListEnvs list envs -func (p *envUsecaseImpl) ListEnvs(ctx context.Context, page, pageSize int, listOption apisv1.ListEnvOptions) ([]*apisv1.Env, error) { +func (p *envUsecaseImpl) ListEnvs(ctx context.Context, page, pageSize int, listOption apisv1.ListEnvOptions) (*apisv1.ListEnvResponse, error) { entities, err := listEnvs(ctx, p.ds, listOption.Project, &datastore.ListOptions{Page: page, PageSize: pageSize, SortBy: []datastore.SortOption{{Key: "createTime", Order: datastore.SortOrderDescending}}}) if err != nil { return nil, err @@ -111,19 +111,23 @@ func (p *envUsecaseImpl) ListEnvs(ctx context.Context, page, pageSize int, listO envs = append(envs, convertEnvModel2Base(ee, Targets)) } - projects, err := listProjects(ctx, p.ds) + projectResp, err := listProjects(ctx, p.ds, 0, 0) if err != nil { return nil, err } for _, e := range envs { - for _, pj := range projects { + for _, pj := range projectResp.Projects { if e.Project.Name == pj.Name { e.Project.Alias = pj.Alias break } } } - return envs, nil + total, err := p.ds.Count(ctx, &model.Env{Project: listOption.Project}, nil) + if err != nil { + return nil, err + } + return &apisv1.ListEnvResponse{Envs: envs, Total: total}, nil } func checkEqual(old, new []string) bool { diff --git a/pkg/apiserver/rest/usecase/project.go b/pkg/apiserver/rest/usecase/project.go index be8390db7..ca66e444d 100644 --- a/pkg/apiserver/rest/usecase/project.go +++ b/pkg/apiserver/rest/usecase/project.go @@ -34,7 +34,7 @@ import ( // ProjectUsecase project manage usecase. type ProjectUsecase interface { GetProject(ctx context.Context, projectName string) (*model.Project, error) - ListProjects(ctx context.Context) ([]*apisv1.ProjectBase, error) + ListProjects(ctx context.Context, page, pageSize int) (*apisv1.ListProjectResponse, error) CreateProject(ctx context.Context, req apisv1.CreateProjectRequest) (*apisv1.ProjectBase, error) } @@ -59,12 +59,12 @@ func NewProjectUsecase(ds datastore.DataStore) ProjectUsecase { func (p *projectUsecaseImpl) initDefaultProjectEnvTarget(defaultNamespace string) { ctx := context.Background() - entities, err := listProjects(ctx, p.ds) + projResp, err := listProjects(ctx, p.ds, 0, 0) if err != nil { log.Logger.Errorf("initialize project failed %v", err) return } - if len(entities) > 0 { + if len(projResp.Projects) > 0 { return } log.Logger.Info("no default project found, adding a default project with default env and target") @@ -128,9 +128,9 @@ func (p *projectUsecaseImpl) GetProject(ctx context.Context, projectName string) return project, nil } -func listProjects(ctx context.Context, ds datastore.DataStore) ([]*apisv1.ProjectBase, error) { +func listProjects(ctx context.Context, ds datastore.DataStore, page, pageSize int) (*apisv1.ListProjectResponse, error) { var project = model.Project{} - entitys, err := ds.List(ctx, &project, &datastore.ListOptions{SortBy: []datastore.SortOption{{Key: "createTime", Order: datastore.SortOrderDescending}}}) + entitys, err := ds.List(ctx, &project, &datastore.ListOptions{Page: page, PageSize: pageSize, SortBy: []datastore.SortOption{{Key: "createTime", Order: datastore.SortOrderDescending}}}) if err != nil { return nil, err } @@ -139,12 +139,16 @@ func listProjects(ctx context.Context, ds datastore.DataStore) ([]*apisv1.Projec project := entity.(*model.Project) projects = append(projects, convertProjectModel2Base(project)) } - return projects, nil + total, err := ds.Count(ctx, &model.Project{}, nil) + if err != nil { + return nil, err + } + return &apisv1.ListProjectResponse{Projects: projects, Total: total}, nil } // ListProjects list projects -func (p *projectUsecaseImpl) ListProjects(ctx context.Context) ([]*apisv1.ProjectBase, error) { - return listProjects(ctx, p.ds) +func (p *projectUsecaseImpl) ListProjects(ctx context.Context, page, pageSize int) (*apisv1.ListProjectResponse, error) { + return listProjects(ctx, p.ds, page, pageSize) } // DeleteProject delete a project diff --git a/pkg/apiserver/rest/usecase/project_test.go b/pkg/apiserver/rest/usecase/project_test.go index c5a39f24b..065eebf4f 100644 --- a/pkg/apiserver/rest/usecase/project_test.go +++ b/pkg/apiserver/rest/usecase/project_test.go @@ -50,10 +50,10 @@ var _ = Describe("Test project usecase functions", func() { err = k8sClient.Create(context.TODO(), &ns) Expect(err).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) projectUsecase = &projectUsecaseImpl{k8sClient: k8sClient, ds: ds} - pp, err := projectUsecase.ListProjects(context.TODO()) + pp, err := projectUsecase.ListProjects(context.TODO(), 0, 0) Expect(err).Should(BeNil()) // reset all projects - for _, p := range pp { + for _, p := range pp.Projects { _ = projectUsecase.DeleteProject(context.TODO(), p.Name) } @@ -61,7 +61,7 @@ var _ = Describe("Test project usecase functions", func() { envs, err := envImpl.ListEnvs(context.TODO(), 0, 0, apisv1.ListEnvOptions{}) Expect(err).Should(BeNil()) // reset all projects - for _, e := range envs { + for _, e := range envs.Envs { _ = envImpl.DeleteEnv(context.TODO(), e.Name) } targetImpl = &targetUsecaseImpl{k8sClient: k8sClient, ds: ds} @@ -129,7 +129,7 @@ var _ = Describe("Test project usecase functions", func() { base, err := projectUsecase.CreateProject(context.TODO(), req) Expect(err).Should(BeNil()) Expect(cmp.Diff(base.Description, req.Description)).Should(BeEmpty()) - _, err = projectUsecase.ListProjects(context.TODO()) + _, err = projectUsecase.ListProjects(context.TODO(), 0, 0) Expect(err).Should(BeNil()) projectUsecase.DeleteProject(context.TODO(), "test-project") }) diff --git a/pkg/apiserver/rest/usecase/target_model.go b/pkg/apiserver/rest/usecase/target_model.go index 714088a48..61c7b87ef 100644 --- a/pkg/apiserver/rest/usecase/target_model.go +++ b/pkg/apiserver/rest/usecase/target_model.go @@ -33,6 +33,9 @@ import ( ) func createTargetNamespace(ctx context.Context, k8sClient client.Client, clusterName, namespace, targetName string) error { + if clusterName == "" || namespace == "" { + return bcode.ErrTargetInvalidWithEmptyClusterOrNamespace + } err := utils.CreateOrUpdateNamespace(multicluster.ContextWithClusterName(ctx, clusterName), k8sClient, namespace, utils.MergeOverrideLabels(map[string]string{ oam.LabelRuntimeNamespaceUsage: oam.VelaNamespaceUsageTarget, }), utils.MergeNoConflictLabels(map[string]string{ diff --git a/pkg/apiserver/rest/utils/bcode/008_target.go b/pkg/apiserver/rest/utils/bcode/008_target.go index 96cd65772..1620814fe 100644 --- a/pkg/apiserver/rest/utils/bcode/008_target.go +++ b/pkg/apiserver/rest/utils/bcode/008_target.go @@ -27,3 +27,6 @@ var ErrTargetInUseCantDeleted = NewBcode(404, 80003, "target in use, can't be de // ErrTargetNamespaceAlreadyBound indicates the namespace already belongs to other target, one namespace can only belong to one target var ErrTargetNamespaceAlreadyBound = NewBcode(400, 80004, "the namespace specified already belongs to other target") + +// ErrTargetInvalidWithEmptyClusterOrNamespace indicates the namespace/cluster of target is empty +var ErrTargetInvalidWithEmptyClusterOrNamespace = NewBcode(400, 80005, "the namespace or cluster of target should not be empty") diff --git a/pkg/apiserver/rest/webservice/env.go b/pkg/apiserver/rest/webservice/env.go index 516da99eb..463656110 100644 --- a/pkg/apiserver/rest/webservice/env.go +++ b/pkg/apiserver/rest/webservice/env.go @@ -93,7 +93,7 @@ func (n *envWebService) list(req *restful.Request, res *restful.Response) { bcode.ReturnError(req, res, err) return } - if err := res.WriteEntity(apis.ListEnvResponse{Envs: envs}); err != nil { + if err := res.WriteEntity(envs); err != nil { bcode.ReturnError(req, res, err) return } diff --git a/pkg/apiserver/rest/webservice/project.go b/pkg/apiserver/rest/webservice/project.go index de1aa90bb..54aebb12c 100644 --- a/pkg/apiserver/rest/webservice/project.go +++ b/pkg/apiserver/rest/webservice/project.go @@ -18,11 +18,12 @@ package webservice import ( restfulspec "github.com/emicklei/go-restful-openapi/v2" - restful "github.com/emicklei/go-restful/v3" + "github.com/emicklei/go-restful/v3" "github.com/oam-dev/kubevela/pkg/apiserver/log" apis "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1" "github.com/oam-dev/kubevela/pkg/apiserver/rest/usecase" + "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils" "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils/bcode" ) @@ -60,12 +61,17 @@ func (n *projectWebService) GetWebService() *restful.WebService { } func (n *projectWebService) listprojects(req *restful.Request, res *restful.Response) { - projects, err := n.projectUsecase.ListProjects(req.Request.Context()) + page, pageSize, err := utils.ExtractPagingParams(req, minPageSize, maxPageSize) if err != nil { bcode.ReturnError(req, res, err) return } - if err := res.WriteEntity(apis.ListProjectResponse{Projects: projects}); err != nil { + projects, err := n.projectUsecase.ListProjects(req.Request.Context(), page, pageSize) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := res.WriteEntity(projects); err != nil { bcode.ReturnError(req, res, err) return } diff --git a/pkg/apiserver/rest/webservice/webservice_suite_test.go b/pkg/apiserver/rest/webservice/webservice_suite_test.go index 342e017b5..9857d5565 100644 --- a/pkg/apiserver/rest/webservice/webservice_suite_test.go +++ b/pkg/apiserver/rest/webservice/webservice_suite_test.go @@ -14,16 +14,88 @@ See the License for the specific language governing permissions and limitations under the License. */ -package webservice_test +package webservice import ( + "context" + "fmt" + "math/rand" "testing" + "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + + "k8s.io/client-go/rest" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + "github.com/oam-dev/kubevela/pkg/apiserver/clients" + "github.com/oam-dev/kubevela/pkg/apiserver/datastore" + "github.com/oam-dev/kubevela/pkg/apiserver/datastore/kubeapi" + "github.com/oam-dev/kubevela/pkg/apiserver/datastore/mongodb" + "github.com/oam-dev/kubevela/pkg/utils/common" ) +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + func TestWebservice(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Webservice Suite") } + +var _ = BeforeSuite(func(done Done) { + rand.Seed(time.Now().UnixNano()) + By("bootstrapping Sync test environment") + + testEnv = &envtest.Environment{ + ControlPlaneStartTimeout: time.Minute * 3, + ControlPlaneStopTimeout: time.Minute, + UseExistingCluster: pointer.BoolPtr(false), + CRDDirectoryPaths: []string{"../../../charts/vela-core/crds"}, + } + + By("start kube test env") + var err error + cfg, err = testEnv.Start() + Expect(err).ShouldNot(HaveOccurred()) + Expect(cfg).ToNot(BeNil()) + + By("new kube client") + cfg.Timeout = time.Minute * 2 + k8sClient, err = client.New(cfg, client.Options{Scheme: common.Scheme}) + Expect(err).Should(BeNil()) + Expect(k8sClient).ToNot(BeNil()) + clients.SetKubeClient(k8sClient) + By("new kube client success") + clients.SetKubeClient(k8sClient) + Expect(err).Should(BeNil()) + close(done) +}, 240) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).ToNot(HaveOccurred()) +}) + +func NewDatastore(cfg datastore.Config) (ds datastore.DataStore, err error) { + switch cfg.Type { + case "mongodb": + ds, err = mongodb.New(context.Background(), cfg) + if err != nil { + return nil, fmt.Errorf("create mongodb datastore instance failure %w", err) + } + case "kubeapi": + ds, err = kubeapi.New(context.Background(), cfg) + if err != nil { + return nil, fmt.Errorf("create mongodb datastore instance failure %w", err) + } + default: + return nil, fmt.Errorf("not support datastore type %s", cfg.Type) + } + return ds, nil +}