diff --git a/server/api/agent.go b/server/api/agent.go index 7b1e9d4cb..7fdeaff5e 100644 --- a/server/api/agent.go +++ b/server/api/agent.go @@ -29,9 +29,9 @@ import ( ) func GetAgents(c *gin.Context) { - agents, err := store.FromContext(c).AgentList() + agents, err := store.FromContext(c).AgentList(session.Pagination(c)) if err != nil { - c.String(500, "Error getting agent list. %s", err) + c.String(http.StatusInternalServerError, "Error getting agent list. %s", err) return } c.JSON(http.StatusOK, agents) @@ -65,7 +65,7 @@ func GetAgentTasks(c *gin.Context) { return } - tasks := []*model.Task{} + var tasks []*model.Task info := server.Config.Services.Queue.Info(c) for _, task := range info.Running { if task.AgentID == agent.ID { diff --git a/server/api/badge.go b/server/api/badge.go index 8d8bfc0a9..bc0f5e4a1 100644 --- a/server/api/badge.go +++ b/server/api/badge.go @@ -29,6 +29,7 @@ import ( "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/badges" "github.com/woodpecker-ci/woodpecker/server/ccmenu" + "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/store" "github.com/woodpecker-ci/woodpecker/server/store/types" ) @@ -72,7 +73,7 @@ func GetCC(c *gin.Context) { return } - pipelines, err := _store.GetPipelineList(repo, 1) + pipelines, err := _store.GetPipelineList(repo, &model.ListOptions{Page: 1, PerPage: 1}) if err != nil || len(pipelines) == 0 { c.AbortWithStatus(http.StatusNotFound) return diff --git a/server/api/cron.go b/server/api/cron.go index 323b73688..78b39dac1 100644 --- a/server/api/cron.go +++ b/server/api/cron.go @@ -187,7 +187,7 @@ func PatchCron(c *gin.Context) { // to the response in json format. func GetCronList(c *gin.Context) { repo := session.Repo(c) - list, err := store.FromContext(c).CronList(repo) + list, err := store.FromContext(c).CronList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting cron list. %s", err) return diff --git a/server/api/file.go b/server/api/file.go index 519903aea..19bf157e7 100644 --- a/server/api/file.go +++ b/server/api/file.go @@ -44,7 +44,7 @@ func FileList(c *gin.Context) { return } - files, err := _store.FileList(pipeline) + files, err := _store.FileList(pipeline, session.Pagination(c)) if err != nil { _ = c.AbortWithError(http.StatusInternalServerError, err) return diff --git a/server/api/global_secret.go b/server/api/global_secret.go index 6822d9b29..49b4f0e91 100644 --- a/server/api/global_secret.go +++ b/server/api/global_secret.go @@ -18,6 +18,7 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/woodpecker-ci/woodpecker/server/router/middleware/session" "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/model" @@ -26,7 +27,7 @@ import ( // GetGlobalSecretList gets the global secret list from // the database and writes to the response in json format. func GetGlobalSecretList(c *gin.Context) { - list, err := server.Config.Services.Secrets.GlobalSecretList() + list, err := server.Config.Services.Secrets.GlobalSecretList(session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting global secret list. %s", err) return diff --git a/server/api/org_secret.go b/server/api/org_secret.go index 2fe5ae9ae..16fcffca2 100644 --- a/server/api/org_secret.go +++ b/server/api/org_secret.go @@ -18,6 +18,7 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/woodpecker-ci/woodpecker/server/router/middleware/session" "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/model" @@ -38,11 +39,11 @@ func GetOrgSecret(c *gin.Context) { c.JSON(http.StatusOK, secret.Copy()) } -// GetOrgSecretList gest the organization secret list from +// GetOrgSecretList gets the organization secret list from // the database and writes to the response in json format. func GetOrgSecretList(c *gin.Context) { owner := c.Param("owner") - list, err := server.Config.Services.Secrets.OrgSecretList(owner) + list, err := server.Config.Services.Secrets.OrgSecretList(owner, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list for %q. %s", owner, err) return diff --git a/server/api/pipeline.go b/server/api/pipeline.go index 4c87c91e8..b6c082f1a 100644 --- a/server/api/pipeline.go +++ b/server/api/pipeline.go @@ -28,16 +28,15 @@ import ( "strconv" "time" - "github.com/woodpecker-ci/woodpecker/server" - "github.com/woodpecker-ci/woodpecker/server/store/types" - "github.com/gin-gonic/gin" "github.com/rs/zerolog/log" + "github.com/woodpecker-ci/woodpecker/server" "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/pipeline" "github.com/woodpecker-ci/woodpecker/server/router/middleware/session" "github.com/woodpecker-ci/woodpecker/server/store" + "github.com/woodpecker-ci/woodpecker/server/store/types" ) func CreatePipeline(c *gin.Context) { @@ -89,13 +88,8 @@ func createTmpPipeline(event model.WebhookEvent, commitSHA string, repo *model.R func GetPipelines(c *gin.Context) { repo := session.Repo(c) - page, err := strconv.Atoi(c.DefaultQuery("page", "1")) - if err != nil { - _ = c.AbortWithError(http.StatusBadRequest, err) - return - } - pipelines, err := store.FromContext(c).GetPipelineList(repo, page) + pipelines, err := store.FromContext(c).GetPipelineList(repo, session.Pagination(c)) if err != nil { if errors.Is(err, types.RecordNotExist) { c.AbortWithStatus(http.StatusNotFound) @@ -130,7 +124,7 @@ func GetPipeline(c *gin.Context) { _ = c.AbortWithError(http.StatusInternalServerError, err) return } - files, _ := _store.FileList(pl) + files, _ := _store.FileList(pl, &model.ListOptions{All: true}) steps, _ := _store.StepList(pl) if pl.Steps, err = model.Tree(steps); err != nil { _ = c.AbortWithError(http.StatusInternalServerError, err) diff --git a/server/api/registry.go b/server/api/registry.go index 0a5862ad3..368d184b3 100644 --- a/server/api/registry.go +++ b/server/api/registry.go @@ -114,7 +114,7 @@ func PatchRegistry(c *gin.Context) { // to the response in json format. func GetRegistryList(c *gin.Context) { repo := session.Repo(c) - list, err := server.Config.Services.Registries.RegistryList(repo) + list, err := server.Config.Services.Registries.RegistryList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting registry list. %s", err) return diff --git a/server/api/repo.go b/server/api/repo.go index 64db2f800..2c0c06f12 100644 --- a/server/api/repo.go +++ b/server/api/repo.go @@ -222,7 +222,7 @@ func GetRepoBranches(c *gin.Context) { user := session.User(c) f := server.Config.Services.Forge - branches, err := f.Branches(c, user, repo) + branches, err := f.Branches(c, user, repo, session.Pagination(c)) if err != nil { _ = c.AbortWithError(http.StatusInternalServerError, err) return @@ -234,10 +234,9 @@ func GetRepoBranches(c *gin.Context) { func GetRepoPullRequests(c *gin.Context) { repo := session.Repo(c) user := session.User(c) - page := session.Pagination(c) f := server.Config.Services.Forge - prs, err := f.PullRequests(c, user, repo, page) + prs, err := f.PullRequests(c, user, repo, session.Pagination(c)) if err != nil { _ = c.AbortWithError(http.StatusInternalServerError, err) return diff --git a/server/api/repo_secret.go b/server/api/repo_secret.go index 612d138cd..05553ab25 100644 --- a/server/api/repo_secret.go +++ b/server/api/repo_secret.go @@ -113,7 +113,7 @@ func PatchSecret(c *gin.Context) { // to the response in json format. func GetSecretList(c *gin.Context) { repo := session.Repo(c) - list, err := server.Config.Services.Secrets.SecretList(repo) + list, err := server.Config.Services.Secrets.SecretList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list. %s", err) return diff --git a/server/api/stream.go b/server/api/stream.go index 34242f615..d965f52a5 100644 --- a/server/api/stream.go +++ b/server/api/stream.go @@ -61,7 +61,7 @@ func EventStreamSSE(c *gin.Context) { user := session.User(c) repo := map[string]bool{} if user != nil { - repos, _ := store.FromContext(c).RepoList(user, false) + repos, _ := store.FromContext(c).RepoList(user, false, true) for _, r := range repos { repo[r.FullName] = true } diff --git a/server/api/user.go b/server/api/user.go index 84d57746c..76657bff0 100644 --- a/server/api/user.go +++ b/server/api/user.go @@ -63,7 +63,7 @@ func GetRepos(c *gin.Context) { user := session.User(c) all, _ := strconv.ParseBool(c.Query("all")) - dbRepos, err := _store.RepoList(user, true) + activeRepos, err := _store.RepoList(user, true, true) if err != nil { c.String(http.StatusInternalServerError, "Error fetching repository list. %s", err) return @@ -71,7 +71,7 @@ func GetRepos(c *gin.Context) { if all { active := map[string]bool{} - for _, r := range dbRepos { + for _, r := range activeRepos { active[r.FullName] = r.IsActive } @@ -94,13 +94,7 @@ func GetRepos(c *gin.Context) { return } - active := make([]*model.Repo, 0) - for _, repo := range dbRepos { - if repo.IsActive { - active = append(active, repo) - } - } - c.JSON(http.StatusOK, active) + c.JSON(http.StatusOK, activeRepos) } func PostToken(c *gin.Context) { diff --git a/server/api/users.go b/server/api/users.go index fd610da00..6152cd734 100644 --- a/server/api/users.go +++ b/server/api/users.go @@ -22,11 +22,12 @@ import ( "github.com/gorilla/securecookie" "github.com/woodpecker-ci/woodpecker/server/model" + "github.com/woodpecker-ci/woodpecker/server/router/middleware/session" "github.com/woodpecker-ci/woodpecker/server/store" ) func GetUsers(c *gin.Context) { - users, err := store.FromContext(c).GetUserList() + users, err := store.FromContext(c).GetUserList(session.Pagination(c)) if err != nil { c.String(500, "Error getting user list. %s", err) return diff --git a/server/forge/bitbucket/bitbucket.go b/server/forge/bitbucket/bitbucket.go index 81c8401bc..18b2b3059 100644 --- a/server/forge/bitbucket/bitbucket.go +++ b/server/forge/bitbucket/bitbucket.go @@ -21,6 +21,7 @@ import ( "net/http" "net/url" + shared_utils "github.com/woodpecker-ci/woodpecker/shared/utils" "golang.org/x/oauth2" "github.com/woodpecker-ci/woodpecker/server" @@ -132,15 +133,18 @@ func (c *config) Refresh(ctx context.Context, user *model.User) (bool, error) { // Teams returns a list of all team membership for the Bitbucket account. func (c *config) Teams(ctx context.Context, u *model.User) ([]*model.Team, error) { - opts := &internal.ListWorkspacesOpts{ - PageLen: 100, - Role: "member", - } - resp, err := c.newClient(ctx, u).ListWorkspaces(opts) - if err != nil { - return nil, err - } - return convertWorkspaceList(resp.Values), nil + return shared_utils.Paginate(func(page int) ([]*model.Team, error) { + opts := &internal.ListWorkspacesOpts{ + PageLen: 100, + Page: page, + Role: "member", + } + resp, err := c.newClient(ctx, u).ListWorkspaces(opts) + if err != nil { + return nil, err + } + return convertWorkspaceList(resp.Values), nil + }) } // Repo returns the named Bitbucket repository. @@ -261,7 +265,7 @@ func (c *config) Netrc(u *model.User, _ *model.Repo) (*model.Netrc, error) { } // Branches returns the names of all branches for the named repository. -func (c *config) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]string, error) { +func (c *config) Branches(ctx context.Context, u *model.User, r *model.Repo, _ *model.ListOptions) ([]string, error) { bitbucketBranches, err := c.newClient(ctx, u).ListBranches(r.Owner, r.Name) if err != nil { return nil, err @@ -280,7 +284,7 @@ func (c *config) BranchHead(_ context.Context, _ *model.User, _ *model.Repo, _ s return "", forge_types.ErrNotImplemented } -func (c *config) PullRequests(_ context.Context, _ *model.User, _ *model.Repo, _ *model.PaginationData) ([]*model.PullRequest, error) { +func (c *config) PullRequests(_ context.Context, _ *model.User, _ *model.Repo, _ *model.ListOptions) ([]*model.PullRequest, error) { return nil, forge_types.ErrNotImplemented } diff --git a/server/forge/bitbucket/fixtures/handler.go b/server/forge/bitbucket/fixtures/handler.go index 8e3381a04..810a41681 100644 --- a/server/forge/bitbucket/fixtures/handler.go +++ b/server/forge/bitbucket/fixtures/handler.go @@ -76,7 +76,11 @@ func getWorkspaces(c *gin.Context) { case "Bearer teams_not_found", "Bearer c81e728d": c.String(404, "") default: - c.String(200, workspacesPayload) + if c.Query("page") == "" || c.Query("page") == "1" { + c.String(200, workspacesPayload) + } else { + c.String(200, "{\"values\":[]}") + } } } diff --git a/server/forge/bitbucketserver/bitbucketserver.go b/server/forge/bitbucketserver/bitbucketserver.go index a67ec6a14..84949f239 100644 --- a/server/forge/bitbucketserver/bitbucketserver.go +++ b/server/forge/bitbucketserver/bitbucketserver.go @@ -229,8 +229,8 @@ func (c *Config) Activate(ctx context.Context, u *model.User, r *model.Repo, lin } // Branches returns the names of all branches for the named repository. -func (c *Config) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]string, error) { - bitbucketBranches, err := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token).ListBranches(r.Owner, r.Name) +func (c *Config) Branches(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]string, error) { + bitbucketBranches, err := internal.NewClientWithToken(ctx, c.URL, c.Consumer, u.Token).ListBranches(r.Owner, r.Name, p.Page, p.PerPage) if err != nil { return nil, err } @@ -248,7 +248,7 @@ func (c *Config) BranchHead(_ context.Context, _ *model.User, _ *model.Repo, _ s return "", forge_types.ErrNotImplemented } -func (c *Config) PullRequests(_ context.Context, _ *model.User, _ *model.Repo, _ *model.PaginationData) ([]*model.PullRequest, error) { +func (c *Config) PullRequests(_ context.Context, _ *model.User, _ *model.Repo, _ *model.ListOptions) ([]*model.PullRequest, error) { return nil, forge_types.ErrNotImplemented } diff --git a/server/forge/bitbucketserver/internal/client.go b/server/forge/bitbucketserver/internal/client.go index 51af542b3..3f3af3614 100644 --- a/server/forge/bitbucketserver/internal/client.go +++ b/server/forge/bitbucketserver/internal/client.go @@ -43,7 +43,7 @@ const ( pathHookEnabled = "%s/rest/api/1.0/projects/%s/repos/%s/settings/hooks/%s/enabled" pathHookSettings = "%s/rest/api/1.0/projects/%s/repos/%s/settings/hooks/%s/settings" pathStatus = "%s/rest/build-status/1.0/commits/%s" - pathBranches = "%s/rest/api/1.0/projects/%s/repos/%s/branches" + pathBranches = "%s/rest/api/1.0/projects/%s/repos/%s/branches?limit=%d&start=%d" ) type Client struct { @@ -323,8 +323,8 @@ func (c *Client) paginatedRepos(start int) ([]*Repo, error) { return repoResponse.Values, nil } -func (c *Client) ListBranches(owner, name string) ([]*Branch, error) { - uri := fmt.Sprintf(pathBranches, c.base, owner, name) +func (c *Client) ListBranches(owner, name string, page, limit int) ([]*Branch, error) { + uri := fmt.Sprintf(pathBranches, c.base, owner, name, limit, limit*(page-1)) response, err := c.doGet(uri) if err != nil { return nil, err diff --git a/server/forge/common/utils.go b/server/forge/common/utils.go index 4249db9b7..da3128f59 100644 --- a/server/forge/common/utils.go +++ b/server/forge/common/utils.go @@ -37,28 +37,3 @@ func ExtractHostFromCloneURL(cloneURL string) (string, error) { return host, nil } - -// Paginate iterates over a func call until it does not return new items and return it as list -func Paginate[T any](get func(page int) ([]T, error)) ([]T, error) { - items := make([]T, 0, 10) - page := 1 - lenFirstBatch := -1 - - for { - batch, err := get(page) - if err != nil { - return nil, err - } - items = append(items, batch...) - - if page == 1 { - lenFirstBatch = len(batch) - } else if len(batch) < lenFirstBatch || len(batch) == 0 { - break - } - - page++ - } - - return items, nil -} diff --git a/server/forge/common/utils_test.go b/server/forge/common/utils_test.go index a426cf737..e9a7dbf51 100644 --- a/server/forge/common/utils_test.go +++ b/server/forge/common/utils_test.go @@ -17,7 +17,6 @@ package common_test import ( "testing" - "github.com/stretchr/testify/assert" "github.com/woodpecker-ci/woodpecker/server/forge/common" ) @@ -31,29 +30,3 @@ func Test_Netrc(t *testing.T) { t.Errorf("Expected host to be git.example.com, got %s", host) } } - -func TestPaginate(t *testing.T) { - apiExec := 0 - apiMock := func(page int) []int { - apiExec++ - switch page { - case 0, 1: - return []int{11, 12, 13} - case 2: - return []int{21, 22, 23} - case 3: - return []int{31, 32} - default: - return []int{} - } - } - - result, _ := common.Paginate(func(page int) ([]int, error) { - return apiMock(page), nil - }) - - assert.EqualValues(t, 3, apiExec) - if assert.Len(t, result, 8) { - assert.EqualValues(t, []int{11, 12, 13, 21, 22, 23, 31, 32}, result) - } -} diff --git a/server/forge/forge.go b/server/forge/forge.go index 971fdc6f3..2bbaaddb7 100644 --- a/server/forge/forge.go +++ b/server/forge/forge.go @@ -27,7 +27,6 @@ import ( ) // TODO: use pagination -// TODO: add Driver() who return source forge back type Forge interface { // Name returns the string name of this driver @@ -73,14 +72,13 @@ type Forge interface { Deactivate(ctx context.Context, u *model.User, r *model.Repo, link string) error // Branches returns the names of all branches for the named repository. - // TODO: Add proper pagination handling and remove workaround in gitea forge - Branches(ctx context.Context, u *model.User, r *model.Repo) ([]string, error) + Branches(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]string, error) // BranchHead returns the sha of the head (latest commit) of the specified branch BranchHead(ctx context.Context, u *model.User, r *model.Repo, branch string) (string, error) // PullRequests returns all pull requests for the named repository. - PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.PaginationData) ([]*model.PullRequest, error) + PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]*model.PullRequest, error) // Hook parses the post-commit hook from the Request body and returns the // required data in a standard format. diff --git a/server/forge/gitea/gitea.go b/server/forge/gitea/gitea.go index 080d3865f..16c108aba 100644 --- a/server/forge/gitea/gitea.go +++ b/server/forge/gitea/gitea.go @@ -41,6 +41,7 @@ import ( forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types" "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/store" + shared_utils "github.com/woodpecker-ci/woodpecker/shared/utils" ) const ( @@ -195,7 +196,7 @@ func (c *Gitea) Teams(ctx context.Context, u *model.User) ([]*model.Team, error) return nil, err } - return common.Paginate(func(page int) ([]*model.Team, error) { + return shared_utils.Paginate(func(page int) ([]*model.Team, error) { orgs, _, err := client.ListMyOrgs( gitea.ListOrgsOptions{ ListOptions: gitea.ListOptions{ @@ -251,7 +252,7 @@ func (c *Gitea) Repos(ctx context.Context, u *model.User) ([]*model.Repo, error) return nil, err } - return common.Paginate(func(page int) ([]*model.Repo, error) { + return shared_utils.Paginate(func(page int) ([]*model.Repo, error) { repos, _, err := client.ListMyRepos( gitea.ListReposOptions{ ListOptions: gitea.ListOptions{ @@ -415,7 +416,7 @@ func (c *Gitea) Deactivate(ctx context.Context, u *model.User, r *model.Repo, li } // Branches returns the names of all branches for the named repository. -func (c *Gitea) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]string, error) { +func (c *Gitea) Branches(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]string, error) { token := "" if u != nil { token = u.Token @@ -425,20 +426,16 @@ func (c *Gitea) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]s return nil, err } - branches, err := common.Paginate(func(page int) ([]string, error) { - branches, _, err := client.ListRepoBranches(r.Owner, r.Name, - gitea.ListRepoBranchesOptions{ListOptions: gitea.ListOptions{Page: page}}) - result := make([]string, len(branches)) - for i := range branches { - result[i] = branches[i].Name - } - return result, err - }) + branches, _, err := client.ListRepoBranches(r.Owner, r.Name, + gitea.ListRepoBranchesOptions{ListOptions: gitea.ListOptions{Page: p.Page, PageSize: p.PerPage}}) if err != nil { return nil, err } - - return branches, nil + result := make([]string, len(branches)) + for i := range branches { + result[i] = branches[i].Name + } + return result, err } // BranchHead returns the sha of the head (latest commit) of the specified branch @@ -460,7 +457,7 @@ func (c *Gitea) BranchHead(ctx context.Context, u *model.User, r *model.Repo, br return b.Commit.ID, nil } -func (c *Gitea) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.PaginationData) ([]*model.PullRequest, error) { +func (c *Gitea) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]*model.PullRequest, error) { token := "" if u != nil { token = u.Token @@ -471,7 +468,7 @@ func (c *Gitea) PullRequests(ctx context.Context, u *model.User, r *model.Repo, } pullRequests, _, err := client.ListRepoPullRequests(r.Owner, r.Name, gitea.ListPullRequestsOptions{ - ListOptions: gitea.ListOptions{Page: int(p.Page), PageSize: int(p.PerPage)}, + ListOptions: gitea.ListOptions{Page: p.Page, PageSize: p.PerPage}, State: gitea.StateOpen, }) if err != nil { @@ -606,7 +603,7 @@ func (c *Gitea) getChangedFilesForPR(ctx context.Context, repo *model.Repo, inde return []string{}, nil } - return common.Paginate(func(page int) ([]string, error) { + return shared_utils.Paginate(func(page int) ([]string, error) { giteaFiles, _, err := client.ListPullRequestFiles(repo.Owner, repo.Name, index, gitea.ListPullRequestFilesOptions{ListOptions: gitea.ListOptions{Page: page}}) if err != nil { diff --git a/server/forge/github/github.go b/server/forge/github/github.go index 34dea4ca5..c2822eaf0 100644 --- a/server/forge/github/github.go +++ b/server/forge/github/github.go @@ -270,7 +270,7 @@ func (c *client) Dir(ctx context.Context, u *model.User, r *model.Repo, b *model return files, nil } -func (c *client) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.PaginationData) ([]*model.PullRequest, error) { +func (c *client) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]*model.PullRequest, error) { token := "" if u != nil { token = u.Token @@ -278,7 +278,7 @@ func (c *client) PullRequests(ctx context.Context, u *model.User, r *model.Repo, client := c.newClientToken(ctx, token) pullRequests, _, err := client.PullRequests.List(ctx, r.Owner, r.Name, &github.PullRequestListOptions{ - ListOptions: github.ListOptions{Page: int(p.Page), PerPage: int(p.PerPage)}, + ListOptions: github.ListOptions{Page: p.Page, PerPage: p.PerPage}, State: "open", }) if err != nil { @@ -504,14 +504,16 @@ func (c *client) Activate(ctx context.Context, u *model.User, r *model.Repo, lin } // Branches returns the names of all branches for the named repository. -func (c *client) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]string, error) { +func (c *client) Branches(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]string, error) { token := "" if u != nil { token = u.Token } client := c.newClientToken(ctx, token) - githubBranches, _, err := client.Repositories.ListBranches(ctx, r.Owner, r.Name, &github.BranchListOptions{}) + githubBranches, _, err := client.Repositories.ListBranches(ctx, r.Owner, r.Name, &github.BranchListOptions{ + ListOptions: github.ListOptions{Page: p.Page, PerPage: p.PerPage}, + }) if err != nil { return nil, err } @@ -571,21 +573,23 @@ func (c *client) loadChangedFilesFromPullRequest(ctx context.Context, pull *gith return nil, err } - opts := &github.ListOptions{Page: 1} - fileList := make([]string, 0, 16) - for opts.Page > 0 { - files, resp, err := c.newClientToken(ctx, user.Token).PullRequests.ListFiles(ctx, repo.Owner, repo.Name, pull.GetNumber(), opts) - if err != nil { - return nil, err + pipeline.ChangedFiles, err = utils.Paginate(func(page int) ([]string, error) { + opts := &github.ListOptions{Page: page} + fileList := make([]string, 0, 16) + for opts.Page > 0 { + files, resp, err := c.newClientToken(ctx, user.Token).PullRequests.ListFiles(ctx, repo.Owner, repo.Name, pull.GetNumber(), opts) + if err != nil { + return nil, err + } + + for _, file := range files { + fileList = append(fileList, file.GetFilename(), file.GetPreviousFilename()) + } + + opts.Page = resp.NextPage } + return utils.DedupStrings(fileList), nil + }) - for _, file := range files { - fileList = append(fileList, file.GetFilename(), file.GetPreviousFilename()) - } - - opts.Page = resp.NextPage - } - pipeline.ChangedFiles = utils.DedupStrings(fileList) - - return pipeline, nil + return pipeline, err } diff --git a/server/forge/gitlab/gitlab.go b/server/forge/gitlab/gitlab.go index e97314679..093e9069a 100644 --- a/server/forge/gitlab/gitlab.go +++ b/server/forge/gitlab/gitlab.go @@ -301,7 +301,7 @@ func (g *GitLab) Repos(ctx context.Context, user *model.User) ([]*model.Repo, er return repos, err } -func (g *GitLab) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.PaginationData) ([]*model.PullRequest, error) { +func (g *GitLab) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]*model.PullRequest, error) { token := "" if u != nil { token = u.Token @@ -318,7 +318,7 @@ func (g *GitLab) PullRequests(ctx context.Context, u *model.User, r *model.Repo, state := "open" pullRequests, _, err := client.MergeRequests.ListProjectMergeRequests(_repo.ID, &gitlab.ListProjectMergeRequestsOptions{ - ListOptions: gitlab.ListOptions{Page: int(p.Page), PerPage: int(p.PerPage)}, + ListOptions: gitlab.ListOptions{Page: p.Page, PerPage: p.PerPage}, State: &state, }) if err != nil { @@ -543,7 +543,7 @@ func (g *GitLab) Deactivate(ctx context.Context, user *model.User, repo *model.R } // Branches returns the names of all branches for the named repository. -func (g *GitLab) Branches(ctx context.Context, user *model.User, repo *model.Repo) ([]string, error) { +func (g *GitLab) Branches(ctx context.Context, user *model.User, repo *model.Repo, p *model.ListOptions) ([]string, error) { token := "" if user != nil { token = user.Token @@ -558,7 +558,9 @@ func (g *GitLab) Branches(ctx context.Context, user *model.User, repo *model.Rep return nil, err } - gitlabBranches, _, err := client.Branches.ListBranches(_repo.ID, &gitlab.ListBranchesOptions{}, gitlab.WithContext(ctx)) + gitlabBranches, _, err := client.Branches.ListBranches(_repo.ID, + &gitlab.ListBranchesOptions{ListOptions: gitlab.ListOptions{Page: p.Page, PerPage: p.PerPage}}, + gitlab.WithContext(ctx)) if err != nil { return nil, err } diff --git a/server/forge/gogs/gogs.go b/server/forge/gogs/gogs.go index 848fde318..6d112bbdd 100644 --- a/server/forge/gogs/gogs.go +++ b/server/forge/gogs/gogs.go @@ -258,7 +258,7 @@ func (c *client) Deactivate(_ context.Context, _ *model.User, _ *model.Repo, _ s } // Branches returns the names of all branches for the named repository. -func (c *client) Branches(_ context.Context, u *model.User, r *model.Repo) ([]string, error) { +func (c *client) Branches(_ context.Context, u *model.User, r *model.Repo, _ *model.ListOptions) ([]string, error) { token := "" if u != nil { token = u.Token @@ -289,7 +289,7 @@ func (c *client) BranchHead(_ context.Context, u *model.User, r *model.Repo, bra return b.Commit.ID, nil } -func (c *client) PullRequests(_ context.Context, _ *model.User, _ *model.Repo, _ *model.PaginationData) ([]*model.PullRequest, error) { +func (c *client) PullRequests(_ context.Context, _ *model.User, _ *model.Repo, _ *model.ListOptions) ([]*model.PullRequest, error) { return nil, forge_types.ErrNotImplemented } diff --git a/server/forge/mocks/forge.go b/server/forge/mocks/forge.go index 864c56fdf..c7c6b6809 100644 --- a/server/forge/mocks/forge.go +++ b/server/forge/mocks/forge.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.23.0. DO NOT EDIT. +// Code generated by mockery v2.23.1. DO NOT EDIT. package mocks @@ -81,25 +81,25 @@ func (_m *Forge) BranchHead(ctx context.Context, u *model.User, r *model.Repo, b return r0, r1 } -// Branches provides a mock function with given fields: ctx, u, r -func (_m *Forge) Branches(ctx context.Context, u *model.User, r *model.Repo) ([]string, error) { - ret := _m.Called(ctx, u, r) +// Branches provides a mock function with given fields: ctx, u, r, p +func (_m *Forge) Branches(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]string, error) { + ret := _m.Called(ctx, u, r, p) var r0 []string var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo) ([]string, error)); ok { - return rf(ctx, u, r) + if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.ListOptions) ([]string, error)); ok { + return rf(ctx, u, r, p) } - if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo) []string); ok { - r0 = rf(ctx, u, r) + if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.ListOptions) []string); ok { + r0 = rf(ctx, u, r, p) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]string) } } - if rf, ok := ret.Get(1).(func(context.Context, *model.User, *model.Repo) error); ok { - r1 = rf(ctx, u, r) + if rf, ok := ret.Get(1).(func(context.Context, *model.User, *model.Repo, *model.ListOptions) error); ok { + r1 = rf(ctx, u, r, p) } else { r1 = ret.Error(1) } @@ -301,15 +301,15 @@ func (_m *Forge) OrgMembership(ctx context.Context, u *model.User, owner string) } // PullRequests provides a mock function with given fields: ctx, u, r, p -func (_m *Forge) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.PaginationData) ([]*model.PullRequest, error) { +func (_m *Forge) PullRequests(ctx context.Context, u *model.User, r *model.Repo, p *model.ListOptions) ([]*model.PullRequest, error) { ret := _m.Called(ctx, u, r, p) var r0 []*model.PullRequest var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.PaginationData) ([]*model.PullRequest, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.ListOptions) ([]*model.PullRequest, error)); ok { return rf(ctx, u, r, p) } - if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.PaginationData) []*model.PullRequest); ok { + if rf, ok := ret.Get(0).(func(context.Context, *model.User, *model.Repo, *model.ListOptions) []*model.PullRequest); ok { r0 = rf(ctx, u, r, p) } else { if ret.Get(0) != nil { @@ -317,7 +317,7 @@ func (_m *Forge) PullRequests(ctx context.Context, u *model.User, r *model.Repo, } } - if rf, ok := ret.Get(1).(func(context.Context, *model.User, *model.Repo, *model.PaginationData) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, *model.User, *model.Repo, *model.ListOptions) error); ok { r1 = rf(ctx, u, r, p) } else { r1 = ret.Error(1) diff --git a/server/grpc/rpc.go b/server/grpc/rpc.go index 2ce718715..575101f13 100644 --- a/server/grpc/rpc.go +++ b/server/grpc/rpc.go @@ -27,9 +27,8 @@ import ( "strconv" "time" - "github.com/rs/zerolog/log" - "github.com/prometheus/client_golang/prometheus" + "github.com/rs/zerolog/log" "google.golang.org/grpc/metadata" grpcMetadata "google.golang.org/grpc/metadata" @@ -141,7 +140,8 @@ func (s *RPC) Update(c context.Context, id string, state rpc.State) error { log.Error().Err(err).Msg("rpc.update: cannot update step") } - if currentPipeline.Steps, err = s.store.StepList(currentPipeline); err != nil { + currentPipeline.Steps, err = s.store.StepList(currentPipeline) + if err != nil { log.Error().Err(err).Msg("can not get step list from store") } if currentPipeline.Steps, err = model.Tree(currentPipeline.Steps); err != nil { diff --git a/server/model/file.go b/server/model/file.go index 5004f3676..9ab876878 100644 --- a/server/model/file.go +++ b/server/model/file.go @@ -19,7 +19,7 @@ import "io" // FileStore persists pipeline artifacts to storage. type FileStore interface { - FileList(*Pipeline) ([]*File, error) + FileList(*Pipeline, *ListOptions) ([]*File, error) FileFind(*Step, string) (*File, error) FileRead(*Step, string) (io.ReadCloser, error) FileCreate(*File, io.Reader) error diff --git a/server/model/pagination.go b/server/model/pagination.go index f28d72a58..936e8ce94 100644 --- a/server/model/pagination.go +++ b/server/model/pagination.go @@ -1,6 +1,20 @@ package model -type PaginationData struct { - Page int64 - PerPage int64 +type ListOptions struct { + All bool + Page int + PerPage int +} + +func ApplyPagination[T any](d *ListOptions, slice []T) []T { + if d.All { + return slice + } + if d.PerPage*(d.Page-1) > len(slice) { + return []T{} + } + if d.PerPage*(d.Page) > len(slice) { + return slice[d.PerPage*(d.Page-1):] + } + return slice[d.PerPage*(d.Page-1) : d.PerPage*(d.Page)] } diff --git a/server/model/pagination_test.go b/server/model/pagination_test.go new file mode 100644 index 000000000..37e0512b2 --- /dev/null +++ b/server/model/pagination_test.go @@ -0,0 +1,20 @@ +package model + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestApplyPagination(t *testing.T) { + example := []int{ + 0, 1, 2, + } + + assert.Equal(t, ApplyPagination(&ListOptions{All: true}, example), example) + assert.Equal(t, ApplyPagination(&ListOptions{Page: 1, PerPage: 1}, example), []int{0}) + assert.Equal(t, ApplyPagination(&ListOptions{Page: 2, PerPage: 2}, example), []int{2}) + assert.Equal(t, ApplyPagination(&ListOptions{Page: 3, PerPage: 1}, example), []int{2}) + assert.Equal(t, ApplyPagination(&ListOptions{Page: 4, PerPage: 1}, example), []int{}) + assert.Equal(t, ApplyPagination(&ListOptions{Page: 5, PerPage: 1}, example), []int{}) +} diff --git a/server/model/registry.go b/server/model/registry.go index 3739a8105..fa4b708a6 100644 --- a/server/model/registry.go +++ b/server/model/registry.go @@ -29,7 +29,7 @@ var ( // RegistryService defines a service for managing registries. type RegistryService interface { RegistryFind(*Repo, string) (*Registry, error) - RegistryList(*Repo) ([]*Registry, error) + RegistryList(*Repo, *ListOptions) ([]*Registry, error) RegistryCreate(*Repo, *Registry) error RegistryUpdate(*Repo, *Registry) error RegistryDelete(*Repo, string) error @@ -38,13 +38,13 @@ type RegistryService interface { // ReadOnlyRegistryService defines a service for managing registries. type ReadOnlyRegistryService interface { RegistryFind(*Repo, string) (*Registry, error) - RegistryList(*Repo) ([]*Registry, error) + RegistryList(*Repo, *ListOptions) ([]*Registry, error) } // RegistryStore persists registry information to storage. type RegistryStore interface { RegistryFind(*Repo, string) (*Registry, error) - RegistryList(*Repo) ([]*Registry, error) + RegistryList(*Repo, *ListOptions) ([]*Registry, error) RegistryCreate(*Registry) error RegistryUpdate(*Registry) error RegistryDelete(repo *Repo, addr string) error diff --git a/server/model/secret.go b/server/model/secret.go index f221c469d..16013f429 100644 --- a/server/model/secret.go +++ b/server/model/secret.go @@ -32,22 +32,22 @@ var ( // SecretService defines a service for managing secrets. type SecretService interface { - SecretListPipeline(*Repo, *Pipeline) ([]*Secret, error) + SecretListPipeline(*Repo, *Pipeline, *ListOptions) ([]*Secret, error) // Repository secrets SecretFind(*Repo, string) (*Secret, error) - SecretList(*Repo) ([]*Secret, error) + SecretList(*Repo, *ListOptions) ([]*Secret, error) SecretCreate(*Repo, *Secret) error SecretUpdate(*Repo, *Secret) error SecretDelete(*Repo, string) error // Organization secrets OrgSecretFind(string, string) (*Secret, error) - OrgSecretList(string) ([]*Secret, error) + OrgSecretList(string, *ListOptions) ([]*Secret, error) OrgSecretCreate(string, *Secret) error OrgSecretUpdate(string, *Secret) error OrgSecretDelete(string, string) error // Global secrets GlobalSecretFind(string) (*Secret, error) - GlobalSecretList() ([]*Secret, error) + GlobalSecretList(*ListOptions) ([]*Secret, error) GlobalSecretCreate(*Secret) error GlobalSecretUpdate(*Secret) error GlobalSecretDelete(string) error @@ -56,14 +56,14 @@ type SecretService interface { // SecretStore persists secret information to storage. type SecretStore interface { SecretFind(*Repo, string) (*Secret, error) - SecretList(*Repo, bool) ([]*Secret, error) + SecretList(*Repo, bool, *ListOptions) ([]*Secret, error) SecretCreate(*Secret) error SecretUpdate(*Secret) error SecretDelete(*Secret) error OrgSecretFind(string, string) (*Secret, error) - OrgSecretList(string) ([]*Secret, error) + OrgSecretList(string, *ListOptions) ([]*Secret, error) GlobalSecretFind(string) (*Secret, error) - GlobalSecretList() ([]*Secret, error) + GlobalSecretList(*ListOptions) ([]*Secret, error) SecretListAll() ([]*Secret, error) } diff --git a/server/pipeline/cancel.go b/server/pipeline/cancel.go index b1325f817..0b8a0c2fc 100644 --- a/server/pipeline/cancel.go +++ b/server/pipeline/cancel.go @@ -123,7 +123,7 @@ func cancelPreviousPipelines( } // get all active activeBuilds - activeBuilds, err := _store.GetActivePipelineList(repo, -1) + activeBuilds, err := _store.GetActivePipelineList(repo) if err != nil { return err } diff --git a/server/pipeline/decline.go b/server/pipeline/decline.go index fe6a5c706..065d77954 100644 --- a/server/pipeline/decline.go +++ b/server/pipeline/decline.go @@ -34,7 +34,8 @@ func Decline(ctx context.Context, store store.Store, pipeline *model.Pipeline, u return nil, fmt.Errorf("error updating pipeline. %w", err) } - if pipeline.Steps, err = store.StepList(pipeline); err != nil { + pipeline.Steps, err = store.StepList(pipeline) + if err != nil { log.Error().Err(err).Msg("can not get step list from store") } if pipeline.Steps, err = model.Tree(pipeline.Steps); err != nil { diff --git a/server/pipeline/items.go b/server/pipeline/items.go index b69524846..3ade29599 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -43,12 +43,12 @@ func createPipelineItems(_ context.Context, store store.Store, log.Error().Err(err).Str("repo", repo.FullName).Msgf("Error getting last pipeline before pipeline number '%d'", currentPipeline.Number) } - secs, err := server.Config.Services.Secrets.SecretListPipeline(repo, currentPipeline) + secs, err := server.Config.Services.Secrets.SecretListPipeline(repo, currentPipeline, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("Error getting secrets for %s#%d", repo.FullName, currentPipeline.Number) } - regs, err := server.Config.Services.Registries.RegistryList(repo) + regs, err := server.Config.Services.Registries.RegistryList(repo, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("Error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number) } diff --git a/server/plugins/encryption/wrapper/store/secret_store.go b/server/plugins/encryption/wrapper/store/secret_store.go index 61a27aed9..36b676991 100644 --- a/server/plugins/encryption/wrapper/store/secret_store.go +++ b/server/plugins/encryption/wrapper/store/secret_store.go @@ -32,8 +32,8 @@ func (wrapper *EncryptedSecretStore) SecretFind(repo *model.Repo, s string) (*mo return result, nil } -func (wrapper *EncryptedSecretStore) SecretList(repo *model.Repo, b bool) ([]*model.Secret, error) { - results, err := wrapper.store.SecretList(repo, b) +func (wrapper *EncryptedSecretStore) SecretList(repo *model.Repo, b bool, p *model.ListOptions) ([]*model.Secret, error) { + results, err := wrapper.store.SecretList(repo, b, p) if err != nil { return nil, err } @@ -112,8 +112,8 @@ func (wrapper *EncryptedSecretStore) OrgSecretFind(s, s2 string) (*model.Secret, return result, nil } -func (wrapper *EncryptedSecretStore) OrgSecretList(s string) ([]*model.Secret, error) { - results, err := wrapper.store.OrgSecretList(s) +func (wrapper *EncryptedSecretStore) OrgSecretList(s string, p *model.ListOptions) ([]*model.Secret, error) { + results, err := wrapper.store.OrgSecretList(s, p) if err != nil { return nil, err } @@ -138,8 +138,8 @@ func (wrapper *EncryptedSecretStore) GlobalSecretFind(s string) (*model.Secret, return result, nil } -func (wrapper *EncryptedSecretStore) GlobalSecretList() ([]*model.Secret, error) { - results, err := wrapper.store.GlobalSecretList() +func (wrapper *EncryptedSecretStore) GlobalSecretList(p *model.ListOptions) ([]*model.Secret, error) { + results, err := wrapper.store.GlobalSecretList(p) if err != nil { return nil, err } diff --git a/server/plugins/registry/combine.go b/server/plugins/registry/combine.go index f5b8ea050..6886fd2e5 100644 --- a/server/plugins/registry/combine.go +++ b/server/plugins/registry/combine.go @@ -30,16 +30,16 @@ func (c combined) RegistryFind(repo *model.Repo, name string) (*model.Registry, return nil, nil } -func (c combined) RegistryList(repo *model.Repo) ([]*model.Registry, error) { +func (c combined) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { var registries []*model.Registry for _, registry := range c.registries { - list, err := registry.RegistryList(repo) + list, err := registry.RegistryList(repo, &model.ListOptions{All: true}) if err != nil { return nil, err } registries = append(registries, list...) } - return registries, nil + return model.ApplyPagination(p, registries), nil } func (c combined) RegistryCreate(repo *model.Repo, registry *model.Registry) error { diff --git a/server/plugins/registry/db.go b/server/plugins/registry/db.go index 0dea6c5d4..cf6b47380 100644 --- a/server/plugins/registry/db.go +++ b/server/plugins/registry/db.go @@ -17,8 +17,8 @@ func (b *db) RegistryFind(repo *model.Repo, name string) (*model.Registry, error return b.store.RegistryFind(repo, name) } -func (b *db) RegistryList(repo *model.Repo) ([]*model.Registry, error) { - return b.store.RegistryList(repo) +func (b *db) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { + return b.store.RegistryList(repo, p) } func (b *db) RegistryCreate(_ *model.Repo, in *model.Registry) error { diff --git a/server/plugins/registry/filesystem.go b/server/plugins/registry/filesystem.go index c525b4637..9b9bb98c1 100644 --- a/server/plugins/registry/filesystem.go +++ b/server/plugins/registry/filesystem.go @@ -75,8 +75,12 @@ func (b *filesystem) RegistryFind(*model.Repo, string) (*model.Registry, error) return nil, nil } -func (b *filesystem) RegistryList(*model.Repo) ([]*model.Registry, error) { - return parseDockerConfig(b.path) +func (b *filesystem) RegistryList(_ *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { + regs, err := parseDockerConfig(b.path) + if err != nil { + return nil, err + } + return model.ApplyPagination(p, regs), nil } // decodeAuth decodes a base64 encoded string and returns username and password diff --git a/server/plugins/secrets/builtin.go b/server/plugins/secrets/builtin.go index 0fff6d2ee..f91db8450 100644 --- a/server/plugins/secrets/builtin.go +++ b/server/plugins/secrets/builtin.go @@ -34,12 +34,12 @@ func (b *builtin) SecretFind(repo *model.Repo, name string) (*model.Secret, erro return b.store.SecretFind(repo, name) } -func (b *builtin) SecretList(repo *model.Repo) ([]*model.Secret, error) { - return b.store.SecretList(repo, false) +func (b *builtin) SecretList(repo *model.Repo, p *model.ListOptions) ([]*model.Secret, error) { + return b.store.SecretList(repo, false, p) } -func (b *builtin) SecretListPipeline(repo *model.Repo, _ *model.Pipeline) ([]*model.Secret, error) { - s, err := b.store.SecretList(repo, true) +func (b *builtin) SecretListPipeline(repo *model.Repo, _ *model.Pipeline, p *model.ListOptions) ([]*model.Secret, error) { + s, err := b.store.SecretList(repo, true, p) if err != nil { return nil, err } @@ -90,8 +90,8 @@ func (b *builtin) OrgSecretFind(owner, name string) (*model.Secret, error) { return b.store.OrgSecretFind(owner, name) } -func (b *builtin) OrgSecretList(owner string) ([]*model.Secret, error) { - return b.store.OrgSecretList(owner) +func (b *builtin) OrgSecretList(owner string, p *model.ListOptions) ([]*model.Secret, error) { + return b.store.OrgSecretList(owner, p) } func (b *builtin) OrgSecretCreate(_ string, in *model.Secret) error { @@ -114,8 +114,8 @@ func (b *builtin) GlobalSecretFind(owner string) (*model.Secret, error) { return b.store.GlobalSecretFind(owner) } -func (b *builtin) GlobalSecretList() ([]*model.Secret, error) { - return b.store.GlobalSecretList() +func (b *builtin) GlobalSecretList(p *model.ListOptions) ([]*model.Secret, error) { + return b.store.GlobalSecretList(p) } func (b *builtin) GlobalSecretCreate(in *model.Secret) error { diff --git a/server/router/middleware/session/pagination.go b/server/router/middleware/session/pagination.go index 412beaf4a..8054199bf 100644 --- a/server/router/middleware/session/pagination.go +++ b/server/router/middleware/session/pagination.go @@ -7,22 +7,19 @@ import ( "github.com/woodpecker-ci/woodpecker/server/model" ) -const ( - defaultPage = 1 - defaultPerPage = 25 -) +const maxPageSize = 50 -func Pagination(c *gin.Context) *model.PaginationData { - page, err := strconv.ParseInt(c.Param("page"), 10, 64) - if err != nil { - page = defaultPage +func Pagination(c *gin.Context) *model.ListOptions { + page, err := strconv.ParseInt(c.Query("page"), 10, 64) + if err != nil || page < 1 { + page = 1 } - perPage, err := strconv.ParseInt(c.Param("perPage"), 10, 64) - if err != nil { - perPage = defaultPerPage + perPage, err := strconv.ParseInt(c.Query("perPage"), 10, 64) + if err != nil || perPage < 1 || perPage > maxPageSize { + perPage = maxPageSize } - return &model.PaginationData{ - Page: page, - PerPage: perPage, + return &model.ListOptions{ + Page: int(page), + PerPage: int(perPage), } } diff --git a/server/store/datastore/agent.go b/server/store/datastore/agent.go index cd65ef27c..a8876200e 100644 --- a/server/store/datastore/agent.go +++ b/server/store/datastore/agent.go @@ -18,9 +18,9 @@ import ( "github.com/woodpecker-ci/woodpecker/server/model" ) -func (s storage) AgentList() ([]*model.Agent, error) { - agents := make([]*model.Agent, 0, 10) - return agents, s.engine.Find(&agents) +func (s storage) AgentList(p *model.ListOptions) ([]*model.Agent, error) { + var agents []*model.Agent + return agents, s.paginate(p).Find(&agents) } func (s storage) AgentFind(id int64) (*model.Agent, error) { diff --git a/server/store/datastore/cron.go b/server/store/datastore/cron.go index edf4be515..1910cafce 100644 --- a/server/store/datastore/cron.go +++ b/server/store/datastore/cron.go @@ -36,9 +36,9 @@ func (s storage) CronFind(repo *model.Repo, id int64) (*model.Cron, error) { return cron, wrapGet(s.engine.Get(cron)) } -func (s storage) CronList(repo *model.Repo) ([]*model.Cron, error) { - crons := make([]*model.Cron, 0, perPage) - return crons, s.engine.Where("repo_id = ?", repo.ID).Find(&crons) +func (s storage) CronList(repo *model.Repo, p *model.ListOptions) ([]*model.Cron, error) { + var crons []*model.Cron + return crons, s.paginate(p).Where("repo_id = ?", repo.ID).Find(&crons) } func (s storage) CronUpdate(_ *model.Repo, cron *model.Cron) error { diff --git a/server/store/datastore/engine.go b/server/store/datastore/engine.go index 648147fb3..a761e316c 100644 --- a/server/store/datastore/engine.go +++ b/server/store/datastore/engine.go @@ -25,9 +25,6 @@ type storage struct { engine *xorm.Engine } -// make sure storage implement Store -var _ store.Store = &storage{} - const perPage = 50 func NewEngine(opts *store.Opts) (store.Store, error) { diff --git a/server/store/datastore/file.go b/server/store/datastore/file.go index 91c6e96ac..66288d45b 100644 --- a/server/store/datastore/file.go +++ b/server/store/datastore/file.go @@ -21,9 +21,10 @@ import ( "github.com/woodpecker-ci/woodpecker/server/model" ) -func (s storage) FileList(pipeline *model.Pipeline) ([]*model.File, error) { - files := make([]*model.File, 0, perPage) - return files, s.engine.Where("file_pipeline_id = ?", pipeline.ID).Find(&files) +func (s storage) FileList(pipeline *model.Pipeline, p *model.ListOptions) ([]*model.File, error) { + var files []*model.File + return files, s.paginate(p).Where("file_pipeline_id = ?", pipeline.ID). + Find(&files) } func (s storage) FileFind(step *model.Step, name string) (*model.File, error) { diff --git a/server/store/datastore/file_test.go b/server/store/datastore/file_test.go index 407e7e296..ce5bbdba0 100644 --- a/server/store/datastore/file_test.go +++ b/server/store/datastore/file_test.go @@ -102,7 +102,7 @@ func TestFileList(t *testing.T) { bytes.NewBufferString("hola mundo"), )) - files, err := store.FileList(&model.Pipeline{ID: 1}) + files, err := store.FileList(&model.Pipeline{ID: 1}, &model.ListOptions{Page: 1, PerPage: 50}) if err != nil { t.Errorf("Unexpected error: select files: %s", err) return diff --git a/server/store/datastore/helper.go b/server/store/datastore/helper.go index 77bcfb390..d6020dd95 100644 --- a/server/store/datastore/helper.go +++ b/server/store/datastore/helper.go @@ -15,6 +15,9 @@ package datastore import ( + "xorm.io/xorm" + + "github.com/woodpecker-ci/woodpecker/server/model" "github.com/woodpecker-ci/woodpecker/server/store/types" ) @@ -39,3 +42,16 @@ func wrapDelete(c int64, err error) error { } return nil } + +func (s storage) paginate(p *model.ListOptions) *xorm.Session { + if p.All { + return s.engine.NewSession() + } + if p.PerPage < 1 { + p.PerPage = 1 + } + if p.Page < 1 { + p.Page = 1 + } + return s.engine.Limit(p.PerPage, p.PerPage*(p.Page-1)) +} diff --git a/server/store/datastore/pipeline.go b/server/store/datastore/pipeline.go index 4ca4c8a88..3403fb4fc 100644 --- a/server/store/datastore/pipeline.go +++ b/server/store/datastore/pipeline.go @@ -72,24 +72,20 @@ func (s storage) GetPipelineLastBefore(repo *model.Repo, branch string, num int6 Get(pipeline)) } -func (s storage) GetPipelineList(repo *model.Repo, page int) ([]*model.Pipeline, error) { - pipelines := make([]*model.Pipeline, 0, perPage) - return pipelines, s.engine.Where("pipeline_repo_id = ?", repo.ID). +func (s storage) GetPipelineList(repo *model.Repo, p *model.ListOptions) ([]*model.Pipeline, error) { + var pipelines []*model.Pipeline + return pipelines, s.paginate(p).Where("pipeline_repo_id = ?", repo.ID). Desc("pipeline_number"). - Limit(perPage, perPage*(page-1)). Find(&pipelines) } // GetActivePipelineList get all pipelines that are pending, running or blocked -func (s storage) GetActivePipelineList(repo *model.Repo, page int) ([]*model.Pipeline, error) { - pipelines := make([]*model.Pipeline, 0, perPage) +func (s storage) GetActivePipelineList(repo *model.Repo) ([]*model.Pipeline, error) { + pipelines := make([]*model.Pipeline, 0) query := s.engine. Where("pipeline_repo_id = ?", repo.ID). In("pipeline_status", model.StatusPending, model.StatusRunning, model.StatusBlocked). Desc("pipeline_number") - if page > 0 { - query = query.Limit(perPage, perPage*(page-1)) - } return pipelines, query.Find(&pipelines) } diff --git a/server/store/datastore/pipeline_test.go b/server/store/datastore/pipeline_test.go index 02722a352..c54ed5e87 100644 --- a/server/store/datastore/pipeline_test.go +++ b/server/store/datastore/pipeline_test.go @@ -292,7 +292,7 @@ func TestPipelines(t *testing.T) { g.Assert(err1).IsNil() err2 := store.CreatePipeline(pipeline2, []*model.Step{}...) g.Assert(err2).IsNil() - pipelines, err3 := store.GetPipelineList(&model.Repo{ID: 1}, 1) + pipelines, err3 := store.GetPipelineList(&model.Repo{ID: 1}, &model.ListOptions{Page: 1, PerPage: 50}) g.Assert(err3).IsNil() g.Assert(len(pipelines)).Equal(2) g.Assert(pipelines[0].ID).Equal(pipeline2.ID) diff --git a/server/store/datastore/registry.go b/server/store/datastore/registry.go index b7da85226..916380b2d 100644 --- a/server/store/datastore/registry.go +++ b/server/store/datastore/registry.go @@ -26,9 +26,9 @@ func (s storage) RegistryFind(repo *model.Repo, addr string) (*model.Registry, e return reg, wrapGet(s.engine.Get(reg)) } -func (s storage) RegistryList(repo *model.Repo) ([]*model.Registry, error) { - regs := make([]*model.Registry, 0, perPage) - return regs, s.engine.OrderBy("registry_id").Where("registry_repo_id = ?", repo.ID).Find(®s) +func (s storage) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { + var regs []*model.Registry + return regs, s.paginate(p).OrderBy("registry_id").Where("registry_repo_id = ?", repo.ID).Find(®s) } func (s storage) RegistryCreate(registry *model.Registry) error { diff --git a/server/store/datastore/registry_test.go b/server/store/datastore/registry_test.go index 4ed8b05dd..83e7fd776 100644 --- a/server/store/datastore/registry_test.go +++ b/server/store/datastore/registry_test.go @@ -82,7 +82,7 @@ func TestRegistryList(t *testing.T) { Password: "bar", })) - list, err := store.RegistryList(&model.Repo{ID: 1}) + list, err := store.RegistryList(&model.Repo{ID: 1}, &model.ListOptions{Page: 1, PerPage: 50}) if err != nil { t.Error(err) return diff --git a/server/store/datastore/repo.go b/server/store/datastore/repo.go index 5a80b8594..bc1a59e68 100644 --- a/server/store/datastore/repo.go +++ b/server/store/datastore/repo.go @@ -140,14 +140,17 @@ func (s storage) DeleteRepo(repo *model.Repo) error { // RepoList list all repos where permissions for specific user are stored // TODO: paginate -func (s storage) RepoList(user *model.User, owned bool) ([]*model.Repo, error) { - repos := make([]*model.Repo, 0, perPage) +func (s storage) RepoList(user *model.User, owned, active bool) ([]*model.Repo, error) { + repos := make([]*model.Repo, 0) sess := s.engine.Table("repos"). Join("INNER", "perms", "perms.perm_repo_id = repos.repo_id"). Where("perms.perm_user_id = ?", user.ID) if owned { sess = sess.And(builder.Eq{"perms.perm_push": true}.Or(builder.Eq{"perms.perm_admin": true})) } + if active { + sess = sess.And(builder.Eq{"repos.repo_active": true}) + } return repos, sess. Asc("repo_full_name"). Find(&repos) diff --git a/server/store/datastore/repo_test.go b/server/store/datastore/repo_test.go index feedf75b5..44f2f908a 100644 --- a/server/store/datastore/repo_test.go +++ b/server/store/datastore/repo_test.go @@ -179,7 +179,7 @@ func TestRepoList(t *testing.T) { assert.NoError(t, store.PermUpsert(perm)) } - repos, err := store.RepoList(user, false) + repos, err := store.RepoList(user, false, false) if err != nil { t.Error(err) return @@ -244,7 +244,7 @@ func TestOwnedRepoList(t *testing.T) { assert.NoError(t, store.PermUpsert(perm)) } - repos, err := store.RepoList(user, true) + repos, err := store.RepoList(user, true, false) if err != nil { t.Error(err) return diff --git a/server/store/datastore/secret.go b/server/store/datastore/secret.go index 2e54d5c5f..7da0f6f9a 100644 --- a/server/store/datastore/secret.go +++ b/server/store/datastore/secret.go @@ -30,14 +30,14 @@ func (s storage) SecretFind(repo *model.Repo, name string) (*model.Secret, error return secret, wrapGet(s.engine.Get(secret)) } -func (s storage) SecretList(repo *model.Repo, includeGlobalAndOrgSecrets bool) ([]*model.Secret, error) { - secrets := make([]*model.Secret, 0, perPage) +func (s storage) SecretList(repo *model.Repo, includeGlobalAndOrgSecrets bool, p *model.ListOptions) ([]*model.Secret, error) { + var secrets []*model.Secret var cond builder.Cond = builder.Eq{"secret_repo_id": repo.ID} if includeGlobalAndOrgSecrets { cond = cond.Or(builder.Eq{"secret_owner": repo.Owner}). Or(builder.And(builder.Eq{"secret_owner": ""}, builder.Eq{"secret_repo_id": 0})) } - return secrets, s.engine.Where(cond).OrderBy(orderSecretsBy).Find(&secrets) + return secrets, s.paginate(p).Where(cond).OrderBy(orderSecretsBy).Find(&secrets) } func (s storage) SecretListAll() ([]*model.Secret, error) { @@ -68,9 +68,9 @@ func (s storage) OrgSecretFind(owner, name string) (*model.Secret, error) { return secret, wrapGet(s.engine.Get(secret)) } -func (s storage) OrgSecretList(owner string) ([]*model.Secret, error) { - secrets := make([]*model.Secret, 0, perPage) - return secrets, s.engine.Where("secret_owner = ?", owner).OrderBy(orderSecretsBy).Find(&secrets) +func (s storage) OrgSecretList(owner string, p *model.ListOptions) ([]*model.Secret, error) { + secrets := make([]*model.Secret, 0) + return secrets, s.paginate(p).Where("secret_owner = ?", owner).OrderBy(orderSecretsBy).Find(&secrets) } func (s storage) GlobalSecretFind(name string) (*model.Secret, error) { @@ -80,7 +80,7 @@ func (s storage) GlobalSecretFind(name string) (*model.Secret, error) { return secret, wrapGet(s.engine.Where(builder.And(builder.Eq{"secret_owner": ""}, builder.Eq{"secret_repo_id": 0})).Get(secret)) } -func (s storage) GlobalSecretList() ([]*model.Secret, error) { - secrets := make([]*model.Secret, 0, perPage) - return secrets, s.engine.Where(builder.And(builder.Eq{"secret_owner": ""}, builder.Eq{"secret_repo_id": 0})).OrderBy(orderSecretsBy).Find(&secrets) +func (s storage) GlobalSecretList(p *model.ListOptions) ([]*model.Secret, error) { + secrets := make([]*model.Secret, 0) + return secrets, s.paginate(p).Where(builder.And(builder.Eq{"secret_owner": ""}, builder.Eq{"secret_repo_id": 0})).OrderBy(orderSecretsBy).Find(&secrets) } diff --git a/server/store/datastore/secret_test.go b/server/store/datastore/secret_test.go index cd78ce942..76398f66d 100644 --- a/server/store/datastore/secret_test.go +++ b/server/store/datastore/secret_test.go @@ -73,7 +73,7 @@ func TestSecretList(t *testing.T) { createTestSecrets(t, store) - list, err := store.SecretList(&model.Repo{ID: 1, Owner: "org"}, false) + list, err := store.SecretList(&model.Repo{ID: 1, Owner: "org"}, false, &model.ListOptions{Page: 1, PerPage: 50}) assert.NoError(t, err) assert.Len(t, list, 2) } @@ -95,7 +95,7 @@ func TestSecretPipelineList(t *testing.T) { createTestSecrets(t, store) - list, err := store.SecretList(&model.Repo{ID: 1, Owner: "org"}, true) + list, err := store.SecretList(&model.Repo{ID: 1, Owner: "org"}, true, &model.ListOptions{Page: 1, PerPage: 50}) assert.NoError(t, err) assert.Len(t, list, 4) } @@ -249,7 +249,7 @@ func TestOrgSecretList(t *testing.T) { createTestSecrets(t, store) - list, err := store.OrgSecretList("org") + list, err := store.OrgSecretList("org", &model.ListOptions{All: true}) assert.NoError(t, err) assert.Len(t, list, 1) @@ -302,7 +302,7 @@ func TestGlobalSecretList(t *testing.T) { createTestSecrets(t, store) - list, err := store.GlobalSecretList() + list, err := store.GlobalSecretList(&model.ListOptions{All: true}) assert.NoError(t, err) assert.Len(t, list, 1) diff --git a/server/store/datastore/step.go b/server/store/datastore/step.go index d833febc2..7258ef870 100644 --- a/server/store/datastore/step.go +++ b/server/store/datastore/step.go @@ -43,7 +43,7 @@ func (s storage) StepChild(pipeline *model.Pipeline, ppid int, child string) (*m } func (s storage) StepList(pipeline *model.Pipeline) ([]*model.Step, error) { - stepList := make([]*model.Step, 0, perPage) + stepList := make([]*model.Step, 0) return stepList, s.engine. Where("step_pipeline_id = ?", pipeline.ID). OrderBy("step_pid"). diff --git a/server/store/datastore/user.go b/server/store/datastore/user.go index dc7e98e48..3f8e84154 100644 --- a/server/store/datastore/user.go +++ b/server/store/datastore/user.go @@ -28,9 +28,9 @@ func (s storage) GetUserLogin(login string) (*model.User, error) { return user, wrapGet(s.engine.Where("user_login=?", login).Get(user)) } -func (s storage) GetUserList() ([]*model.User, error) { - users := make([]*model.User, 0, 10) - return users, s.engine.OrderBy("user_id").Find(&users) +func (s storage) GetUserList(p *model.ListOptions) ([]*model.User, error) { + var users []*model.User + return users, s.paginate(p).OrderBy("user_id").Find(&users) } func (s storage) GetUserCount() (int64, error) { diff --git a/server/store/datastore/users_test.go b/server/store/datastore/users_test.go index ebac577a0..38cc4e0cc 100644 --- a/server/store/datastore/users_test.go +++ b/server/store/datastore/users_test.go @@ -132,7 +132,7 @@ func TestUsers(t *testing.T) { } g.Assert(store.CreateUser(&user1)).IsNil() g.Assert(store.CreateUser(&user2)).IsNil() - users, err := store.GetUserList() + users, err := store.GetUserList(&model.ListOptions{Page: 1, PerPage: 50}) g.Assert(err).IsNil() g.Assert(len(users)).Equal(2) g.Assert(users[0].Login).Equal(user1.Login) diff --git a/server/store/mocks/store.go b/server/store/mocks/store.go index 8190d08c9..4987aaf27 100644 --- a/server/store/mocks/store.go +++ b/server/store/mocks/store.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.23.0. DO NOT EDIT. +// Code generated by mockery v2.23.1. DO NOT EDIT. package mocks @@ -94,25 +94,25 @@ func (_m *Store) AgentFindByToken(_a0 string) (*model.Agent, error) { return r0, r1 } -// AgentList provides a mock function with given fields: -func (_m *Store) AgentList() ([]*model.Agent, error) { - ret := _m.Called() +// AgentList provides a mock function with given fields: p +func (_m *Store) AgentList(p *model.ListOptions) ([]*model.Agent, error) { + ret := _m.Called(p) var r0 []*model.Agent var r1 error - if rf, ok := ret.Get(0).(func() ([]*model.Agent, error)); ok { - return rf() + if rf, ok := ret.Get(0).(func(*model.ListOptions) ([]*model.Agent, error)); ok { + return rf(p) } - if rf, ok := ret.Get(0).(func() []*model.Agent); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(*model.ListOptions) []*model.Agent); ok { + r0 = rf(p) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.Agent) } } - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() + if rf, ok := ret.Get(1).(func(*model.ListOptions) error); ok { + r1 = rf(p) } else { r1 = ret.Error(1) } @@ -379,25 +379,25 @@ func (_m *Store) CronGetLock(_a0 *model.Cron, _a1 int64) (bool, error) { return r0, r1 } -// CronList provides a mock function with given fields: _a0 -func (_m *Store) CronList(_a0 *model.Repo) ([]*model.Cron, error) { - ret := _m.Called(_a0) +// CronList provides a mock function with given fields: _a0, _a1 +func (_m *Store) CronList(_a0 *model.Repo, _a1 *model.ListOptions) ([]*model.Cron, error) { + ret := _m.Called(_a0, _a1) var r0 []*model.Cron var r1 error - if rf, ok := ret.Get(0).(func(*model.Repo) ([]*model.Cron, error)); ok { - return rf(_a0) + if rf, ok := ret.Get(0).(func(*model.Repo, *model.ListOptions) ([]*model.Cron, error)); ok { + return rf(_a0, _a1) } - if rf, ok := ret.Get(0).(func(*model.Repo) []*model.Cron); ok { - r0 = rf(_a0) + if rf, ok := ret.Get(0).(func(*model.Repo, *model.ListOptions) []*model.Cron); ok { + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.Cron) } } - if rf, ok := ret.Get(1).(func(*model.Repo) error); ok { - r1 = rf(_a0) + if rf, ok := ret.Get(1).(func(*model.Repo, *model.ListOptions) error); ok { + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } @@ -513,25 +513,25 @@ func (_m *Store) FileFind(_a0 *model.Step, _a1 string) (*model.File, error) { return r0, r1 } -// FileList provides a mock function with given fields: _a0 -func (_m *Store) FileList(_a0 *model.Pipeline) ([]*model.File, error) { - ret := _m.Called(_a0) +// FileList provides a mock function with given fields: _a0, _a1 +func (_m *Store) FileList(_a0 *model.Pipeline, _a1 *model.ListOptions) ([]*model.File, error) { + ret := _m.Called(_a0, _a1) var r0 []*model.File var r1 error - if rf, ok := ret.Get(0).(func(*model.Pipeline) ([]*model.File, error)); ok { - return rf(_a0) + if rf, ok := ret.Get(0).(func(*model.Pipeline, *model.ListOptions) ([]*model.File, error)); ok { + return rf(_a0, _a1) } - if rf, ok := ret.Get(0).(func(*model.Pipeline) []*model.File); ok { - r0 = rf(_a0) + if rf, ok := ret.Get(0).(func(*model.Pipeline, *model.ListOptions) []*model.File); ok { + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.File) } } - if rf, ok := ret.Get(1).(func(*model.Pipeline) error); ok { - r1 = rf(_a0) + if rf, ok := ret.Get(1).(func(*model.Pipeline, *model.ListOptions) error); ok { + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } @@ -565,25 +565,25 @@ func (_m *Store) FileRead(_a0 *model.Step, _a1 string) (io.ReadCloser, error) { return r0, r1 } -// GetActivePipelineList provides a mock function with given fields: repo, page -func (_m *Store) GetActivePipelineList(repo *model.Repo, page int) ([]*model.Pipeline, error) { - ret := _m.Called(repo, page) +// GetActivePipelineList provides a mock function with given fields: repo +func (_m *Store) GetActivePipelineList(repo *model.Repo) ([]*model.Pipeline, error) { + ret := _m.Called(repo) var r0 []*model.Pipeline var r1 error - if rf, ok := ret.Get(0).(func(*model.Repo, int) ([]*model.Pipeline, error)); ok { - return rf(repo, page) + if rf, ok := ret.Get(0).(func(*model.Repo) ([]*model.Pipeline, error)); ok { + return rf(repo) } - if rf, ok := ret.Get(0).(func(*model.Repo, int) []*model.Pipeline); ok { - r0 = rf(repo, page) + if rf, ok := ret.Get(0).(func(*model.Repo) []*model.Pipeline); ok { + r0 = rf(repo) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.Pipeline) } } - if rf, ok := ret.Get(1).(func(*model.Repo, int) error); ok { - r1 = rf(repo, page) + if rf, ok := ret.Get(1).(func(*model.Repo) error); ok { + r1 = rf(repo) } else { r1 = ret.Error(1) } @@ -720,15 +720,15 @@ func (_m *Store) GetPipelineLastBefore(_a0 *model.Repo, _a1 string, _a2 int64) ( } // GetPipelineList provides a mock function with given fields: _a0, _a1 -func (_m *Store) GetPipelineList(_a0 *model.Repo, _a1 int) ([]*model.Pipeline, error) { +func (_m *Store) GetPipelineList(_a0 *model.Repo, _a1 *model.ListOptions) ([]*model.Pipeline, error) { ret := _m.Called(_a0, _a1) var r0 []*model.Pipeline var r1 error - if rf, ok := ret.Get(0).(func(*model.Repo, int) ([]*model.Pipeline, error)); ok { + if rf, ok := ret.Get(0).(func(*model.Repo, *model.ListOptions) ([]*model.Pipeline, error)); ok { return rf(_a0, _a1) } - if rf, ok := ret.Get(0).(func(*model.Repo, int) []*model.Pipeline); ok { + if rf, ok := ret.Get(0).(func(*model.Repo, *model.ListOptions) []*model.Pipeline); ok { r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { @@ -736,7 +736,7 @@ func (_m *Store) GetPipelineList(_a0 *model.Repo, _a1 int) ([]*model.Pipeline, e } } - if rf, ok := ret.Get(1).(func(*model.Repo, int) error); ok { + if rf, ok := ret.Get(1).(func(*model.Repo, *model.ListOptions) error); ok { r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) @@ -1027,25 +1027,25 @@ func (_m *Store) GetUserCount() (int64, error) { return r0, r1 } -// GetUserList provides a mock function with given fields: -func (_m *Store) GetUserList() ([]*model.User, error) { - ret := _m.Called() +// GetUserList provides a mock function with given fields: p +func (_m *Store) GetUserList(p *model.ListOptions) ([]*model.User, error) { + ret := _m.Called(p) var r0 []*model.User var r1 error - if rf, ok := ret.Get(0).(func() ([]*model.User, error)); ok { - return rf() + if rf, ok := ret.Get(0).(func(*model.ListOptions) ([]*model.User, error)); ok { + return rf(p) } - if rf, ok := ret.Get(0).(func() []*model.User); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(*model.ListOptions) []*model.User); ok { + r0 = rf(p) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.User) } } - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() + if rf, ok := ret.Get(1).(func(*model.ListOptions) error); ok { + r1 = rf(p) } else { r1 = ret.Error(1) } @@ -1105,25 +1105,25 @@ func (_m *Store) GlobalSecretFind(_a0 string) (*model.Secret, error) { return r0, r1 } -// GlobalSecretList provides a mock function with given fields: -func (_m *Store) GlobalSecretList() ([]*model.Secret, error) { - ret := _m.Called() +// GlobalSecretList provides a mock function with given fields: _a0 +func (_m *Store) GlobalSecretList(_a0 *model.ListOptions) ([]*model.Secret, error) { + ret := _m.Called(_a0) var r0 []*model.Secret var r1 error - if rf, ok := ret.Get(0).(func() ([]*model.Secret, error)); ok { - return rf() + if rf, ok := ret.Get(0).(func(*model.ListOptions) ([]*model.Secret, error)); ok { + return rf(_a0) } - if rf, ok := ret.Get(0).(func() []*model.Secret); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(*model.ListOptions) []*model.Secret); ok { + r0 = rf(_a0) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.Secret) } } - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() + if rf, ok := ret.Get(1).(func(*model.ListOptions) error); ok { + r1 = rf(_a0) } else { r1 = ret.Error(1) } @@ -1235,25 +1235,25 @@ func (_m *Store) OrgSecretFind(_a0 string, _a1 string) (*model.Secret, error) { return r0, r1 } -// OrgSecretList provides a mock function with given fields: _a0 -func (_m *Store) OrgSecretList(_a0 string) ([]*model.Secret, error) { - ret := _m.Called(_a0) +// OrgSecretList provides a mock function with given fields: _a0, _a1 +func (_m *Store) OrgSecretList(_a0 string, _a1 *model.ListOptions) ([]*model.Secret, error) { + ret := _m.Called(_a0, _a1) var r0 []*model.Secret var r1 error - if rf, ok := ret.Get(0).(func(string) ([]*model.Secret, error)); ok { - return rf(_a0) + if rf, ok := ret.Get(0).(func(string, *model.ListOptions) ([]*model.Secret, error)); ok { + return rf(_a0, _a1) } - if rf, ok := ret.Get(0).(func(string) []*model.Secret); ok { - r0 = rf(_a0) + if rf, ok := ret.Get(0).(func(string, *model.ListOptions) []*model.Secret); ok { + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.Secret) } } - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(_a0) + if rf, ok := ret.Get(1).(func(string, *model.ListOptions) error); ok { + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } @@ -1411,25 +1411,25 @@ func (_m *Store) RegistryFind(_a0 *model.Repo, _a1 string) (*model.Registry, err return r0, r1 } -// RegistryList provides a mock function with given fields: _a0 -func (_m *Store) RegistryList(_a0 *model.Repo) ([]*model.Registry, error) { - ret := _m.Called(_a0) +// RegistryList provides a mock function with given fields: _a0, _a1 +func (_m *Store) RegistryList(_a0 *model.Repo, _a1 *model.ListOptions) ([]*model.Registry, error) { + ret := _m.Called(_a0, _a1) var r0 []*model.Registry var r1 error - if rf, ok := ret.Get(0).(func(*model.Repo) ([]*model.Registry, error)); ok { - return rf(_a0) + if rf, ok := ret.Get(0).(func(*model.Repo, *model.ListOptions) ([]*model.Registry, error)); ok { + return rf(_a0, _a1) } - if rf, ok := ret.Get(0).(func(*model.Repo) []*model.Registry); ok { - r0 = rf(_a0) + if rf, ok := ret.Get(0).(func(*model.Repo, *model.ListOptions) []*model.Registry); ok { + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.Registry) } } - if rf, ok := ret.Get(1).(func(*model.Repo) error); ok { - r1 = rf(_a0) + if rf, ok := ret.Get(1).(func(*model.Repo, *model.ListOptions) error); ok { + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } @@ -1451,25 +1451,25 @@ func (_m *Store) RegistryUpdate(_a0 *model.Registry) error { return r0 } -// RepoList provides a mock function with given fields: user, owned -func (_m *Store) RepoList(user *model.User, owned bool) ([]*model.Repo, error) { - ret := _m.Called(user, owned) +// RepoList provides a mock function with given fields: user, owned, active +func (_m *Store) RepoList(user *model.User, owned bool, active bool) ([]*model.Repo, error) { + ret := _m.Called(user, owned, active) var r0 []*model.Repo var r1 error - if rf, ok := ret.Get(0).(func(*model.User, bool) ([]*model.Repo, error)); ok { - return rf(user, owned) + if rf, ok := ret.Get(0).(func(*model.User, bool, bool) ([]*model.Repo, error)); ok { + return rf(user, owned, active) } - if rf, ok := ret.Get(0).(func(*model.User, bool) []*model.Repo); ok { - r0 = rf(user, owned) + if rf, ok := ret.Get(0).(func(*model.User, bool, bool) []*model.Repo); ok { + r0 = rf(user, owned, active) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.Repo) } } - if rf, ok := ret.Get(1).(func(*model.User, bool) error); ok { - r1 = rf(user, owned) + if rf, ok := ret.Get(1).(func(*model.User, bool, bool) error); ok { + r1 = rf(user, owned, active) } else { r1 = ret.Error(1) } @@ -1557,25 +1557,25 @@ func (_m *Store) SecretFind(_a0 *model.Repo, _a1 string) (*model.Secret, error) return r0, r1 } -// SecretList provides a mock function with given fields: _a0, _a1 -func (_m *Store) SecretList(_a0 *model.Repo, _a1 bool) ([]*model.Secret, error) { - ret := _m.Called(_a0, _a1) +// SecretList provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Store) SecretList(_a0 *model.Repo, _a1 bool, _a2 *model.ListOptions) ([]*model.Secret, error) { + ret := _m.Called(_a0, _a1, _a2) var r0 []*model.Secret var r1 error - if rf, ok := ret.Get(0).(func(*model.Repo, bool) ([]*model.Secret, error)); ok { - return rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(*model.Repo, bool, *model.ListOptions) ([]*model.Secret, error)); ok { + return rf(_a0, _a1, _a2) } - if rf, ok := ret.Get(0).(func(*model.Repo, bool) []*model.Secret); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(*model.Repo, bool, *model.ListOptions) []*model.Secret); ok { + r0 = rf(_a0, _a1, _a2) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]*model.Secret) } } - if rf, ok := ret.Get(1).(func(*model.Repo, bool) error); ok { - r1 = rf(_a0, _a1) + if rf, ok := ret.Get(1).(func(*model.Repo, bool, *model.ListOptions) error); ok { + r1 = rf(_a0, _a1, _a2) } else { r1 = ret.Error(1) } diff --git a/server/store/store.go b/server/store/store.go index 2a0c71298..4e79089fb 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -32,8 +32,7 @@ type Store interface { // GetUserLogin gets a user by unique Login name. GetUserLogin(string) (*model.User, error) // GetUserList gets a list of all users in the system. - // TODO: paginate - GetUserList() ([]*model.User, error) + GetUserList(p *model.ListOptions) ([]*model.User, error) // GetUserCount gets a count of all users in the system. GetUserCount() (int64, error) // CreateUser creates a new user account. @@ -83,10 +82,9 @@ type Store interface { // GetPipelineLastBefore gets the last pipeline before pipeline number N. GetPipelineLastBefore(*model.Repo, string, int64) (*model.Pipeline, error) // GetPipelineList gets a list of pipelines for the repository - // TODO: paginate - GetPipelineList(*model.Repo, int) ([]*model.Pipeline, error) - // GetPipelineList gets a list of the active pipelines for the repository - GetActivePipelineList(repo *model.Repo, page int) ([]*model.Pipeline, error) + GetPipelineList(*model.Repo, *model.ListOptions) ([]*model.Pipeline, error) + // GetActivePipelineList gets a list of the active pipelines for the repository + GetActivePipelineList(repo *model.Repo) ([]*model.Pipeline, error) // GetPipelineQueue gets a list of pipelines in queue. GetPipelineQueue() ([]*model.Feed, error) // GetPipelineCount gets a count of all pipelines in the system. @@ -100,8 +98,7 @@ type Store interface { UserFeed(*model.User) ([]*model.Feed, error) // Repositories - // RepoList TODO: paginate - RepoList(user *model.User, owned bool) ([]*model.Repo, error) + RepoList(user *model.User, owned, active bool) ([]*model.Repo, error) RepoListLatest(*model.User) ([]*model.Feed, error) // Permissions @@ -119,19 +116,19 @@ type Store interface { // Secrets SecretFind(*model.Repo, string) (*model.Secret, error) - SecretList(*model.Repo, bool) ([]*model.Secret, error) + SecretList(*model.Repo, bool, *model.ListOptions) ([]*model.Secret, error) SecretListAll() ([]*model.Secret, error) SecretCreate(*model.Secret) error SecretUpdate(*model.Secret) error SecretDelete(*model.Secret) error OrgSecretFind(string, string) (*model.Secret, error) - OrgSecretList(string) ([]*model.Secret, error) + OrgSecretList(string, *model.ListOptions) ([]*model.Secret, error) GlobalSecretFind(string) (*model.Secret, error) - GlobalSecretList() ([]*model.Secret, error) + GlobalSecretList(*model.ListOptions) ([]*model.Secret, error) // Registries RegistryFind(*model.Repo, string) (*model.Registry, error) - RegistryList(*model.Repo) ([]*model.Registry, error) + RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) RegistryCreate(*model.Registry) error RegistryUpdate(*model.Registry) error RegistryDelete(repo *model.Repo, addr string) error @@ -152,7 +149,7 @@ type Store interface { LogSave(*model.Step, io.Reader) error // Files - FileList(*model.Pipeline) ([]*model.File, error) + FileList(*model.Pipeline, *model.ListOptions) ([]*model.File, error) FileFind(*model.Step, string) (*model.File, error) FileRead(*model.Step, string) (io.ReadCloser, error) FileCreate(*model.File, io.Reader) error @@ -171,7 +168,7 @@ type Store interface { // Cron CronCreate(*model.Cron) error CronFind(*model.Repo, int64) (*model.Cron, error) - CronList(*model.Repo) ([]*model.Cron, error) + CronList(*model.Repo, *model.ListOptions) ([]*model.Cron, error) CronUpdate(*model.Repo, *model.Cron) error CronDelete(*model.Repo, int64) error CronListNextExecute(int64, int64) ([]*model.Cron, error) @@ -181,7 +178,7 @@ type Store interface { AgentCreate(*model.Agent) error AgentFind(int64) (*model.Agent, error) AgentFindByToken(string) (*model.Agent, error) - AgentList() ([]*model.Agent, error) + AgentList(p *model.ListOptions) ([]*model.Agent, error) AgentUpdate(*model.Agent) error AgentDelete(*model.Agent) error diff --git a/shared/utils/paginate.go b/shared/utils/paginate.go new file mode 100644 index 000000000..bf8178f88 --- /dev/null +++ b/shared/utils/paginate.go @@ -0,0 +1,26 @@ +package utils + +// Paginate iterates over a func call until it does not return new items and return it as list +func Paginate[T any](get func(page int) ([]T, error)) ([]T, error) { + items := make([]T, 0, 10) + page := 1 + lenFirstBatch := -1 + + for { + batch, err := get(page) + if err != nil { + return nil, err + } + items = append(items, batch...) + + if page == 1 { + lenFirstBatch = len(batch) + } else if len(batch) < lenFirstBatch || len(batch) == 0 { + break + } + + page++ + } + + return items, nil +} diff --git a/shared/utils/paginate_test.go b/shared/utils/paginate_test.go new file mode 100644 index 000000000..cc675514d --- /dev/null +++ b/shared/utils/paginate_test.go @@ -0,0 +1,33 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPaginate(t *testing.T) { + apiExec := 0 + apiMock := func(page int) []int { + apiExec++ + switch page { + case 0, 1: + return []int{11, 12, 13} + case 2: + return []int{21, 22, 23} + case 3: + return []int{31, 32} + default: + return []int{} + } + } + + result, _ := Paginate(func(page int) ([]int, error) { + return apiMock(page), nil + }) + + assert.EqualValues(t, 3, apiExec) + if assert.Len(t, result, 8) { + assert.EqualValues(t, []int{11, 12, 13, 21, 22, 23, 31, 32}, result) + } +} diff --git a/web/src/App.vue b/web/src/App.vue index bd81ec59c..7636425ff 100644 --- a/web/src/App.vue +++ b/web/src/App.vue @@ -4,7 +4,7 @@ diff --git a/web/src/views/repo/RepoPullRequests.vue b/web/src/views/repo/RepoPullRequests.vue index 72de50de6..6bdea1f03 100644 --- a/web/src/views/repo/RepoPullRequests.vue +++ b/web/src/views/repo/RepoPullRequests.vue @@ -16,33 +16,29 @@