Feat: add page index for env list API (#3482)

* Feat: add page index for env list API

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

* Fix: add check for target create without namespace or name

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

* Fix: adress commnets

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

docs/apidoc/swagger.json

@@ -6359,12 +6359,12 @@
 		},
 		"v1.ApplicationDeployResponse": {
 			"required": [
-				"createTime",
-				"note",
-				"version",
 				"status",
+				"createTime",
+				"version",
 				"envName",
-				"triggerType"
+				"triggerType",
+				"note"
 			],
 			"properties": {
 				"codeInfo": {
@@ -7420,10 +7420,10 @@
 		},
 		"v1.DetailAddonResponse": {
 			"required": [
-				"invisible",
 				"name",
-				"description",
+				"invisible",
 				"version",
+				"description",
 				"icon",
 				"schema",
 				"uiSchema",
@@ -7497,11 +7497,11 @@
 		},
 		"v1.DetailApplicationResponse": {
 			"required": [
-				"icon",
-				"name",
 				"alias",
 				"project",
 				"description",
+				"icon",
+				"name",
 				"createTime",
 				"updateTime",
 				"policies",
@@ -7568,20 +7568,20 @@
 		},
 		"v1.DetailClusterResponse": {
 			"required": [
-				"name",
-				"alias",
-				"labels",
-				"dashboardURL",
-				"kubeConfigSecret",
 				"createTime",
-				"updateTime",
-				"reason",
-				"icon",
-				"provider",
 				"description",
 				"status",
 				"apiServerURL",
+				"updateTime",
+				"alias",
+				"reason",
+				"name",
+				"icon",
+				"labels",
+				"provider",
+				"dashboardURL",
 				"kubeConfig",
+				"kubeConfigSecret",
 				"resourceInfo"
 			],
 			"properties": {
@@ -7639,14 +7639,14 @@
 		},
 		"v1.DetailComponentResponse": {
 			"required": [
-				"updateTime",
 				"creator",
 				"name",
 				"alias",
 				"type",
-				"main",
 				"createTime",
 				"appPrimaryKey",
+				"main",
+				"updateTime",
 				"definition"
 			],
 			"properties": {
@@ -7748,13 +7748,13 @@
 		},
 		"v1.DetailPolicyResponse": {
 			"required": [
+				"name",
 				"type",
 				"description",
 				"creator",
 				"properties",
 				"createTime",
-				"updateTime",
-				"name"
+				"updateTime"
 			],
 			"properties": {
 				"createTime": {
@@ -7784,16 +7784,16 @@
 		},
 		"v1.DetailRevisionResponse": {
 			"required": [
-				"note",
-				"workflowName",
-				"createTime",
-				"triggerType",
-				"updateTime",
 				"appPrimaryKey",
-				"deployUser",
-				"envName",
 				"version",
 				"status",
+				"workflowName",
+				"updateTime",
+				"deployUser",
+				"note",
+				"triggerType",
+				"createTime",
+				"envName",
 				"reason"
 			],
 			"properties": {
@@ -7848,8 +7848,8 @@
 		},
 		"v1.DetailTargetResponse": {
 			"required": [
-				"updateTime",
 				"createTime",
+				"updateTime",
 				"name"
 			],
 			"properties": {
@@ -7887,11 +7887,11 @@
 		},
 		"v1.DetailUserResponse": {
 			"required": [
+				"email",
 				"disabled",
 				"createTime",
 				"lastLoginTime",
 				"name",
-				"email",
 				"projects"
 			],
 			"properties": {
@@ -7925,12 +7925,12 @@
 		},
 		"v1.DetailWorkflowRecordResponse": {
 			"required": [
-				"status",
-				"name",
 				"namespace",
 				"workflowName",
 				"workflowAlias",
 				"applicationRevision",
+				"status",
+				"name",
 				"deployTime",
 				"deployUser",
 				"note",
@@ -7982,14 +7982,14 @@
 		},
 		"v1.DetailWorkflowResponse": {
 			"required": [
-				"description",
 				"default",
 				"envName",
-				"createTime",
+				"updateTime",
 				"name",
 				"alias",
+				"description",
 				"enable",
-				"updateTime"
+				"createTime"
 			],
 			"properties": {
 				"alias": {
@@ -8348,7 +8348,8 @@
 		},
 		"v1.ListEnvResponse": {
 			"required": [
-				"envs"
+				"envs",
+				"total"
 			],
 			"properties": {
 				"envs": {
@@ -8356,6 +8357,10 @@
 					"items": {
 						"$ref": "#/definitions/v1.Env"
 					}
+				},
+				"total": {
+					"type": "integer",
+					"format": "int64"
 				}
 			}
 		},
@@ -8374,7 +8379,8 @@
 		},
 		"v1.ListProjectResponse": {
 			"required": [
-				"projects"
+				"projects",
+				"total"
 			],
 			"properties": {
 				"projects": {
@@ -8382,6 +8388,10 @@
 					"items": {
 						"$ref": "#/definitions/v1.ProjectBase"
 					}
+				},
+				"total": {
+					"type": "integer",
+					"format": "int64"
 				}
 			}
 		},
@@ -8662,11 +8672,11 @@
 		},
 		"v1.SystemInfoResponse": {
 			"required": [
+				"createTime",
 				"updateTime",
 				"installID",
 				"enableCollection",
 				"loginType",
-				"createTime",
 				"systemVersion"
 			],
 			"properties": {

pkg/apiserver/rest/apis/v1/types.go

@@ -668,6 +668,7 @@ type ApplicationTemplateVersion struct {
 // ListProjectResponse list project response body
 type ListProjectResponse struct {
 	Projects []*ProjectBase `json:"projects"`
+	Total    int64          `json:"total"`
 }
 
 // ProjectBase project base model
@@ -712,7 +713,8 @@ type ListEnvOptions struct {
 
 // ListEnvResponse response the while env list
 type ListEnvResponse struct {
-	Envs []*Env `json:"envs"`
+	Envs  []*Env `json:"envs"`
+	Total int64  `json:"total"`
 }
 
 // CreateEnvRequest contains the env data as request body

pkg/apiserver/rest/usecase/env.go

@@ -38,7 +38,7 @@ import (
 // EnvUsecase defines the API of Env.
 type EnvUsecase interface {
 	GetEnv(ctx context.Context, envName string) (*model.Env, error)
-	ListEnvs(ctx context.Context, page, pageSize int, listOption apisv1.ListEnvOptions) ([]*apisv1.Env, error)
+	ListEnvs(ctx context.Context, page, pageSize int, listOption apisv1.ListEnvOptions) (*apisv1.ListEnvResponse, error)
 	DeleteEnv(ctx context.Context, envName string) error
 	CreateEnv(ctx context.Context, req apisv1.CreateEnvRequest) (*apisv1.Env, error)
 	UpdateEnv(ctx context.Context, envName string, req apisv1.UpdateEnvRequest) (*apisv1.Env, error)
@@ -95,7 +95,7 @@ func (p *envUsecaseImpl) DeleteEnv(ctx context.Context, envName string) error {
 }
 
 // ListEnvs list envs
-func (p *envUsecaseImpl) ListEnvs(ctx context.Context, page, pageSize int, listOption apisv1.ListEnvOptions) ([]*apisv1.Env, error) {
+func (p *envUsecaseImpl) ListEnvs(ctx context.Context, page, pageSize int, listOption apisv1.ListEnvOptions) (*apisv1.ListEnvResponse, error) {
 	entities, err := listEnvs(ctx, p.ds, listOption.Project, &datastore.ListOptions{Page: page, PageSize: pageSize, SortBy: []datastore.SortOption{{Key: "createTime", Order: datastore.SortOrderDescending}}})
 	if err != nil {
 		return nil, err
@@ -111,19 +111,23 @@ func (p *envUsecaseImpl) ListEnvs(ctx context.Context, page, pageSize int, listO
 		envs = append(envs, convertEnvModel2Base(ee, Targets))
 	}
 
-	projects, err := listProjects(ctx, p.ds)
+	projectResp, err := listProjects(ctx, p.ds, 0, 0)
 	if err != nil {
 		return nil, err
 	}
 	for _, e := range envs {
-		for _, pj := range projects {
+		for _, pj := range projectResp.Projects {
 			if e.Project.Name == pj.Name {
 				e.Project.Alias = pj.Alias
 				break
 			}
 		}
 	}
-	return envs, nil
+	total, err := p.ds.Count(ctx, &model.Env{Project: listOption.Project}, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &apisv1.ListEnvResponse{Envs: envs, Total: total}, nil
 }
 
 func checkEqual(old, new []string) bool {

pkg/apiserver/rest/usecase/project.go

@@ -34,7 +34,7 @@ import (
 // ProjectUsecase project manage usecase.
 type ProjectUsecase interface {
 	GetProject(ctx context.Context, projectName string) (*model.Project, error)
-	ListProjects(ctx context.Context) ([]*apisv1.ProjectBase, error)
+	ListProjects(ctx context.Context, page, pageSize int) (*apisv1.ListProjectResponse, error)
 	CreateProject(ctx context.Context, req apisv1.CreateProjectRequest) (*apisv1.ProjectBase, error)
 }
 
@@ -59,12 +59,12 @@ func NewProjectUsecase(ds datastore.DataStore) ProjectUsecase {
 func (p *projectUsecaseImpl) initDefaultProjectEnvTarget(defaultNamespace string) {
 
 	ctx := context.Background()
-	entities, err := listProjects(ctx, p.ds)
+	projResp, err := listProjects(ctx, p.ds, 0, 0)
 	if err != nil {
 		log.Logger.Errorf("initialize project failed %v", err)
 		return
 	}
-	if len(entities) > 0 {
+	if len(projResp.Projects) > 0 {
 		return
 	}
 	log.Logger.Info("no default project found, adding a default project with default env and target")
@@ -128,9 +128,9 @@ func (p *projectUsecaseImpl) GetProject(ctx context.Context, projectName string)
 	return project, nil
 }
 
-func listProjects(ctx context.Context, ds datastore.DataStore) ([]*apisv1.ProjectBase, error) {
+func listProjects(ctx context.Context, ds datastore.DataStore, page, pageSize int) (*apisv1.ListProjectResponse, error) {
 	var project = model.Project{}
-	entitys, err := ds.List(ctx, &project, &datastore.ListOptions{SortBy: []datastore.SortOption{{Key: "createTime", Order: datastore.SortOrderDescending}}})
+	entitys, err := ds.List(ctx, &project, &datastore.ListOptions{Page: page, PageSize: pageSize, SortBy: []datastore.SortOption{{Key: "createTime", Order: datastore.SortOrderDescending}}})
 	if err != nil {
 		return nil, err
 	}
@@ -139,12 +139,16 @@ func listProjects(ctx context.Context, ds datastore.DataStore) ([]*apisv1.Projec
 		project := entity.(*model.Project)
 		projects = append(projects, convertProjectModel2Base(project))
 	}
-	return projects, nil
+	total, err := ds.Count(ctx, &model.Project{}, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &apisv1.ListProjectResponse{Projects: projects, Total: total}, nil
 }
 
 // ListProjects list projects
-func (p *projectUsecaseImpl) ListProjects(ctx context.Context) ([]*apisv1.ProjectBase, error) {
-	return listProjects(ctx, p.ds)
+func (p *projectUsecaseImpl) ListProjects(ctx context.Context, page, pageSize int) (*apisv1.ListProjectResponse, error) {
+	return listProjects(ctx, p.ds, page, pageSize)
 }
 
 // DeleteProject delete a project

pkg/apiserver/rest/usecase/project_test.go

@@ -50,10 +50,10 @@ var _ = Describe("Test project usecase functions", func() {
 		err = k8sClient.Create(context.TODO(), &ns)
 		Expect(err).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
 		projectUsecase = &projectUsecaseImpl{k8sClient: k8sClient, ds: ds}
-		pp, err := projectUsecase.ListProjects(context.TODO())
+		pp, err := projectUsecase.ListProjects(context.TODO(), 0, 0)
 		Expect(err).Should(BeNil())
 		// reset all projects
-		for _, p := range pp {
+		for _, p := range pp.Projects {
 			_ = projectUsecase.DeleteProject(context.TODO(), p.Name)
 		}
 
@@ -61,7 +61,7 @@ var _ = Describe("Test project usecase functions", func() {
 		envs, err := envImpl.ListEnvs(context.TODO(), 0, 0, apisv1.ListEnvOptions{})
 		Expect(err).Should(BeNil())
 		// reset all projects
-		for _, e := range envs {
+		for _, e := range envs.Envs {
 			_ = envImpl.DeleteEnv(context.TODO(), e.Name)
 		}
 		targetImpl = &targetUsecaseImpl{k8sClient: k8sClient, ds: ds}
@@ -129,7 +129,7 @@ var _ = Describe("Test project usecase functions", func() {
 		base, err := projectUsecase.CreateProject(context.TODO(), req)
 		Expect(err).Should(BeNil())
 		Expect(cmp.Diff(base.Description, req.Description)).Should(BeEmpty())
-		_, err = projectUsecase.ListProjects(context.TODO())
+		_, err = projectUsecase.ListProjects(context.TODO(), 0, 0)
 		Expect(err).Should(BeNil())
 		projectUsecase.DeleteProject(context.TODO(), "test-project")
 	})

pkg/apiserver/rest/usecase/target_model.go

@@ -33,6 +33,9 @@ import (
 )
 
 func createTargetNamespace(ctx context.Context, k8sClient client.Client, clusterName, namespace, targetName string) error {
+	if clusterName == "" || namespace == "" {
+		return bcode.ErrTargetInvalidWithEmptyClusterOrNamespace
+	}
 	err := utils.CreateOrUpdateNamespace(multicluster.ContextWithClusterName(ctx, clusterName), k8sClient, namespace, utils.MergeOverrideLabels(map[string]string{
 		oam.LabelRuntimeNamespaceUsage: oam.VelaNamespaceUsageTarget,
 	}), utils.MergeNoConflictLabels(map[string]string{

pkg/apiserver/rest/utils/bcode/008_target.go

@@ -27,3 +27,6 @@ var ErrTargetInUseCantDeleted = NewBcode(404, 80003, "target in use, can't be de
 
 // ErrTargetNamespaceAlreadyBound indicates the namespace already belongs to other target, one namespace can only belong to one target
 var ErrTargetNamespaceAlreadyBound = NewBcode(400, 80004, "the namespace specified already belongs to other target")
+
+// ErrTargetInvalidWithEmptyClusterOrNamespace indicates the namespace/cluster of target is empty
+var ErrTargetInvalidWithEmptyClusterOrNamespace = NewBcode(400, 80005, "the namespace or cluster of target should not be empty")

pkg/apiserver/rest/webservice/env.go

@@ -93,7 +93,7 @@ func (n *envWebService) list(req *restful.Request, res *restful.Response) {
 		bcode.ReturnError(req, res, err)
 		return
 	}
-	if err := res.WriteEntity(apis.ListEnvResponse{Envs: envs}); err != nil {
+	if err := res.WriteEntity(envs); err != nil {
 		bcode.ReturnError(req, res, err)
 		return
 	}

pkg/apiserver/rest/webservice/project.go

@@ -18,11 +18,12 @@ package webservice
 
 import (
 	restfulspec "github.com/emicklei/go-restful-openapi/v2"
-	restful "github.com/emicklei/go-restful/v3"
+	"github.com/emicklei/go-restful/v3"
 
 	"github.com/oam-dev/kubevela/pkg/apiserver/log"
 	apis "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1"
 	"github.com/oam-dev/kubevela/pkg/apiserver/rest/usecase"
+	"github.com/oam-dev/kubevela/pkg/apiserver/rest/utils"
 	"github.com/oam-dev/kubevela/pkg/apiserver/rest/utils/bcode"
 )
 
@@ -60,12 +61,17 @@ func (n *projectWebService) GetWebService() *restful.WebService {
 }
 
 func (n *projectWebService) listprojects(req *restful.Request, res *restful.Response) {
-	projects, err := n.projectUsecase.ListProjects(req.Request.Context())
+	page, pageSize, err := utils.ExtractPagingParams(req, minPageSize, maxPageSize)
 	if err != nil {
 		bcode.ReturnError(req, res, err)
 		return
 	}
-	if err := res.WriteEntity(apis.ListProjectResponse{Projects: projects}); err != nil {
+	projects, err := n.projectUsecase.ListProjects(req.Request.Context(), page, pageSize)
+	if err != nil {
+		bcode.ReturnError(req, res, err)
+		return
+	}
+	if err := res.WriteEntity(projects); err != nil {
 		bcode.ReturnError(req, res, err)
 		return
 	}

pkg/apiserver/rest/webservice/webservice_suite_test.go

@@ -14,16 +14,88 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package webservice_test
+package webservice
 
 import (
+	"context"
+	"fmt"
+	"math/rand"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+
+	"github.com/oam-dev/kubevela/pkg/apiserver/clients"
+	"github.com/oam-dev/kubevela/pkg/apiserver/datastore"
+	"github.com/oam-dev/kubevela/pkg/apiserver/datastore/kubeapi"
+	"github.com/oam-dev/kubevela/pkg/apiserver/datastore/mongodb"
+	"github.com/oam-dev/kubevela/pkg/utils/common"
 )
 
+var cfg *rest.Config
+var k8sClient client.Client
+var testEnv *envtest.Environment
+
 func TestWebservice(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Webservice Suite")
 }
+
+var _ = BeforeSuite(func(done Done) {
+	rand.Seed(time.Now().UnixNano())
+	By("bootstrapping Sync test environment")
+
+	testEnv = &envtest.Environment{
+		ControlPlaneStartTimeout: time.Minute * 3,
+		ControlPlaneStopTimeout:  time.Minute,
+		UseExistingCluster:       pointer.BoolPtr(false),
+		CRDDirectoryPaths:        []string{"../../../charts/vela-core/crds"},
+	}
+
+	By("start kube test env")
+	var err error
+	cfg, err = testEnv.Start()
+	Expect(err).ShouldNot(HaveOccurred())
+	Expect(cfg).ToNot(BeNil())
+
+	By("new kube client")
+	cfg.Timeout = time.Minute * 2
+	k8sClient, err = client.New(cfg, client.Options{Scheme: common.Scheme})
+	Expect(err).Should(BeNil())
+	Expect(k8sClient).ToNot(BeNil())
+	clients.SetKubeClient(k8sClient)
+	By("new kube client success")
+	clients.SetKubeClient(k8sClient)
+	Expect(err).Should(BeNil())
+	close(done)
+}, 240)
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	err := testEnv.Stop()
+	Expect(err).ToNot(HaveOccurred())
+})
+
+func NewDatastore(cfg datastore.Config) (ds datastore.DataStore, err error) {
+	switch cfg.Type {
+	case "mongodb":
+		ds, err = mongodb.New(context.Background(), cfg)
+		if err != nil {
+			return nil, fmt.Errorf("create mongodb datastore instance failure %w", err)
+		}
+	case "kubeapi":
+		ds, err = kubeapi.New(context.Background(), cfg)
+		if err != nil {
+			return nil, fmt.Errorf("create mongodb datastore instance failure %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("not support datastore type %s", cfg.Type)
+	}
+	return ds, nil
+}