From 81d479aedf85685f89a4df27cabcbc2829336917 Mon Sep 17 00:00:00 2001 From: wyike Date: Fri, 22 Apr 2022 10:18:28 +0800 Subject: [PATCH] Fix: change systemInfo some fields (#3715) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add some field an calculate workflow step Signed-off-by: 楚岳 * fix the calculate job cannot start issue Signed-off-by: 楚岳 * fix comments Signed-off-by: 楚岳 fix test Signed-off-by: 楚岳 --- pkg/apiserver/collect/system_info_collect.go | 71 ++++++++++++++++--- .../collect/system_info_collect_test.go | 31 +++++++- pkg/apiserver/model/system_info.go | 14 ++-- pkg/apiserver/rest/apis/v1/types.go | 16 +++-- pkg/apiserver/rest/rest_server.go | 3 +- pkg/apiserver/rest/usecase/system_info.go | 18 ++--- test/e2e-apiserver-test/system_info_test.go | 20 +++--- 7 files changed, 128 insertions(+), 45 deletions(-) diff --git a/pkg/apiserver/collect/system_info_collect.go b/pkg/apiserver/collect/system_info_collect.go index 7326e985a..20ad1b224 100644 --- a/pkg/apiserver/collect/system_info_collect.go +++ b/pkg/apiserver/collect/system_info_collect.go @@ -126,7 +126,7 @@ func (i InfoCalculateCronJob) run() error { func (i InfoCalculateCronJob) calculateAndUpdate(ctx context.Context, systemInfo model.SystemInfo) error { - appCount, topKComp, topKTrait, err := i.calculateAppInfo(ctx) + appCount, topKComp, topKTrait, topWorkflowStep, topKPolicy, err := i.calculateAppInfo(ctx) if err != nil { return err } @@ -142,12 +142,14 @@ func (i InfoCalculateCronJob) calculateAndUpdate(ctx context.Context, systemInfo } statisticInfo := model.StatisticInfo{ - AppCount: genCountInfo(appCount), - TopKCompDef: topKComp, - TopKTraitDef: topKTrait, - ClusterCount: genCountInfo(clusterCount), - EnabledAddon: enabledAddon, - UpdateTime: time.Now(), + AppCount: genCountInfo(appCount), + TopKCompDef: topKComp, + TopKTraitDef: topKTrait, + TopKWorkflowStepDef: topWorkflowStep, + TopKPolicyDef: topKPolicy, + ClusterCount: genClusterCountInfo(clusterCount), + EnabledAddon: enabledAddon, + UpdateTime: time.Now(), } systemInfo.StatisticInfo = statisticInfo @@ -157,16 +159,18 @@ func (i InfoCalculateCronJob) calculateAndUpdate(ctx context.Context, systemInfo return nil } -func (i InfoCalculateCronJob) calculateAppInfo(ctx context.Context) (int, []string, []string, error) { +func (i InfoCalculateCronJob) calculateAppInfo(ctx context.Context) (int, []string, []string, []string, []string, error) { var err error var appCount int compDef := map[string]int{} traitDef := map[string]int{} + workflowDef := map[string]int{} + policyDef := map[string]int{} var app = model.Application{} entities, err := i.ds.List(ctx, &app, &datastore.ListOptions{}) if err != nil { - return 0, nil, nil, err + return 0, nil, nil, nil, nil, err } for _, entity := range entities { appModel, ok := entity.(*model.Application) @@ -179,7 +183,7 @@ func (i InfoCalculateCronJob) calculateAppInfo(ctx context.Context) (int, []stri } comps, err := i.ds.List(ctx, &comp, &datastore.ListOptions{}) if err != nil { - return 0, nil, nil, err + return 0, nil, nil, nil, nil, err } for _, e := range comps { c, ok := e.(*model.ApplicationComponent) @@ -191,9 +195,41 @@ func (i InfoCalculateCronJob) calculateAppInfo(ctx context.Context) (int, []stri traitDef[t.Type]++ } } + + workflow := model.Workflow{ + AppPrimaryKey: app.PrimaryKey(), + } + workflows, err := i.ds.List(ctx, &workflow, &datastore.ListOptions{}) + if err != nil { + return 0, nil, nil, nil, nil, err + } + for _, e := range workflows { + w, ok := e.(*model.Workflow) + if !ok { + continue + } + for _, step := range w.Steps { + workflowDef[step.Type]++ + } + } + + policy := model.ApplicationPolicy{ + AppPrimaryKey: app.PrimaryKey(), + } + policies, err := i.ds.List(ctx, &policy, &datastore.ListOptions{}) + if err != nil { + return 0, nil, nil, nil, nil, err + } + for _, e := range policies { + p, ok := e.(*model.ApplicationPolicy) + if !ok { + continue + } + policyDef[p.Type]++ + } } - return appCount, topKFrequent(compDef, TopKFrequent), topKFrequent(traitDef, TopKFrequent), nil + return appCount, topKFrequent(compDef, TopKFrequent), topKFrequent(traitDef, TopKFrequent), topKFrequent(workflowDef, TopKFrequent), topKFrequent(policyDef, TopKFrequent), nil } func (i InfoCalculateCronJob) calculateAddonInfo(ctx context.Context) (map[string]string, error) { @@ -280,3 +316,16 @@ func genCountInfo(num int) string { return ">=10000" } } + +func genClusterCountInfo(num int) string { + switch { + case num < 3: + return "<3" + case num < 10: + return "<10" + case num < 50: + return "<50" + default: + return ">=50" + } +} diff --git a/pkg/apiserver/collect/system_info_collect_test.go b/pkg/apiserver/collect/system_info_collect_test.go index ee4fcc153..161c65432 100644 --- a/pkg/apiserver/collect/system_info_collect_test.go +++ b/pkg/apiserver/collect/system_info_collect_test.go @@ -90,7 +90,7 @@ var _ = Describe("Test calculate cronJob", func() { }) It("Test calculate app Info", func() { - appNum, topKCom, topKTrait, err := i.calculateAppInfo(ctx) + appNum, topKCom, topKTrait, _, _, err := i.calculateAppInfo(ctx) Expect(err).Should(BeNil()) Expect(appNum).Should(BeEquivalentTo(2)) Expect(topKCom).Should(BeEquivalentTo([]string{"webservice", "helm"})) @@ -131,7 +131,7 @@ var _ = Describe("Test calculate cronJob", func() { Expect(ok).Should(BeTrue()) Expect(info.InstallID).Should(BeEquivalentTo("test-id")) Expect(info.StatisticInfo.AppCount).Should(BeEquivalentTo("<10")) - Expect(info.StatisticInfo.ClusterCount).Should(BeEquivalentTo("<10")) + Expect(info.StatisticInfo.ClusterCount).Should(BeEquivalentTo("<3")) Expect(info.StatisticInfo.TopKCompDef).Should(BeEquivalentTo([]string{"webservice", "helm"})) Expect(info.StatisticInfo.TopKTraitDef).Should(BeEquivalentTo([]string{"rollout", "patch", "expose"})) Expect(info.StatisticInfo.EnabledAddon).Should(BeEquivalentTo(map[string]string{ @@ -193,6 +193,33 @@ func TestGenCountInfo(t *testing.T) { } } +func TestGenClusterCountInfo(t *testing.T) { + testcases := []struct { + count int + res string + }{ + { + count: 2, + res: "<3", + }, + { + count: 7, + res: "<10", + }, + { + count: 34, + res: "<50", + }, + { + count: 100, + res: ">=50", + }, + } + for _, testcase := range testcases { + assert.Equal(t, genClusterCountInfo(testcase.count), testcase.res) + } +} + func TestTopKFrequent(t *testing.T) { testCases := []struct { def map[string]int diff --git a/pkg/apiserver/model/system_info.go b/pkg/apiserver/model/system_info.go index debf7b576..ea30938fc 100644 --- a/pkg/apiserver/model/system_info.go +++ b/pkg/apiserver/model/system_info.go @@ -51,12 +51,14 @@ type DexConfig struct { // StatisticInfo the system statistic info type StatisticInfo struct { - ClusterCount string `json:"clusterCount,omitempty"` - AppCount string `json:"appCount,omitempty"` - EnabledAddon map[string]string `json:"enabledAddon,omitempty"` - TopKCompDef []string `json:"topKCompDef,omitempty"` - TopKTraitDef []string `json:"topKTraitDef,omitempty"` - UpdateTime time.Time `json:"updateTime,omitempty"` + ClusterCount string `json:"clusterCount,omitempty"` + AppCount string `json:"appCount,omitempty"` + EnabledAddon map[string]string `json:"enabledAddon,omitempty"` + TopKCompDef []string `json:"topKCompDef,omitempty"` + TopKTraitDef []string `json:"topKTraitDef,omitempty"` + TopKWorkflowStepDef []string `json:"topKWorkflowStepDef,omitempty"` + TopKPolicyDef []string `json:"topKPolicyDef,omitempty"` + UpdateTime time.Time `json:"updateTime,omitempty"` } // DexStorage dex storage diff --git a/pkg/apiserver/rest/apis/v1/types.go b/pkg/apiserver/rest/apis/v1/types.go index 5deb47d9b..6948e47a4 100644 --- a/pkg/apiserver/rest/apis/v1/types.go +++ b/pkg/apiserver/rest/apis/v1/types.go @@ -1124,19 +1124,21 @@ type SystemInfoResponse struct { // SystemInfo system info type SystemInfo struct { - InstallID string `json:"installID"` + PlatformID string `json:"platformID"` EnableCollection bool `json:"enableCollection"` LoginType string `json:"loginType"` } // StatisticInfo generated by cronJob running in backend type StatisticInfo struct { - ClusterCount string `json:"clusterCount,omitempty"` - AppCount string `json:"appCount,omitempty"` - EnabledAddon map[string]string `json:"enabledAddon,omitempty"` - TopKCompDef []string `json:"topKCompDef,omitempty"` - TopKTraitDef []string `json:"topKTraitDef,omitempty"` - UpdateTime time.Time `json:"updateTime,omitempty"` + ClusterCount string `json:"clusterCount,omitempty"` + AppCount string `json:"appCount,omitempty"` + EnableAddonList map[string]string `json:"enableAddonList,omitempty"` + ComponentDefinitionTopList []string `json:"componentDefinitionTopList,omitempty"` + TraitDefinitionTopList []string `json:"traitDefinitionTopList,omitempty"` + WorkflowDefinitionTopList []string `json:"workflowDefinitionTopList,omitempty"` + PolicyDefinitionTopList []string `json:"policyDefinitionTopList,omitempty"` + UpdateTime time.Time `json:"updateTime,omitempty"` } // SystemInfoRequest request by update SystemInfo diff --git a/pkg/apiserver/rest/rest_server.go b/pkg/apiserver/rest/rest_server.go index aa224561c..a5c4f36ad 100644 --- a/pkg/apiserver/rest/rest_server.go +++ b/pkg/apiserver/rest/rest_server.go @@ -146,10 +146,11 @@ func (s *restServer) setupLeaderElection() (*leaderelection.LeaderElectionConfig Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: func(ctx context.Context) { go velasync.Start(ctx, s.dataStore, restCfg, s.usecases) - s.runWorkflowRecordSync(ctx, s.cfg.LeaderConfig.Duration) if !s.cfg.DisableStatisticCronJob { collect.StartCalculatingInfoCronJob(s.dataStore) } + // this process would block the whole process, any other handler should start before this func + s.runWorkflowRecordSync(ctx, s.cfg.LeaderConfig.Duration) }, OnStoppedLeading: func() { klog.Infof("leader lost: %s", s.cfg.LeaderConfig.ID) diff --git a/pkg/apiserver/rest/usecase/system_info.go b/pkg/apiserver/rest/usecase/system_info.go index 84d759e8b..abcd674e5 100644 --- a/pkg/apiserver/rest/usecase/system_info.go +++ b/pkg/apiserver/rest/usecase/system_info.go @@ -101,12 +101,14 @@ func (u systemInfoUsecaseImpl) GetSystemInfo(ctx context.Context) (*v1.SystemInf GitVersion: version.GitRevision, }, StatisticInfo: v1.StatisticInfo{ - AppCount: info.StatisticInfo.AppCount, - ClusterCount: info.StatisticInfo.ClusterCount, - EnabledAddon: info.StatisticInfo.EnabledAddon, - TopKCompDef: info.StatisticInfo.TopKCompDef, - TopKTraitDef: info.StatisticInfo.TopKTraitDef, - UpdateTime: info.StatisticInfo.UpdateTime, + AppCount: info.StatisticInfo.AppCount, + ClusterCount: info.StatisticInfo.ClusterCount, + EnableAddonList: info.StatisticInfo.EnabledAddon, + ComponentDefinitionTopList: info.StatisticInfo.TopKCompDef, + TraitDefinitionTopList: info.StatisticInfo.TopKTraitDef, + WorkflowDefinitionTopList: info.StatisticInfo.TopKWorkflowStepDef, + PolicyDefinitionTopList: info.StatisticInfo.TopKPolicyDef, + UpdateTime: info.StatisticInfo.UpdateTime, }, }, nil } @@ -144,7 +146,7 @@ func (u systemInfoUsecaseImpl) UpdateSystemInfo(ctx context.Context, sysInfo v1. } return &v1.SystemInfoResponse{ SystemInfo: v1.SystemInfo{ - InstallID: modifiedInfo.InstallID, + PlatformID: modifiedInfo.InstallID, EnableCollection: modifiedInfo.EnableCollection, LoginType: modifiedInfo.LoginType, }, @@ -164,7 +166,7 @@ func (u systemInfoUsecaseImpl) Init(ctx context.Context) error { func convertInfoToBase(info *model.SystemInfo) v1.SystemInfo { return v1.SystemInfo{ - InstallID: info.InstallID, + PlatformID: info.InstallID, EnableCollection: info.EnableCollection, LoginType: info.LoginType, } diff --git a/test/e2e-apiserver-test/system_info_test.go b/test/e2e-apiserver-test/system_info_test.go index 763674c3b..defd69cb6 100644 --- a/test/e2e-apiserver-test/system_info_test.go +++ b/test/e2e-apiserver-test/system_info_test.go @@ -29,16 +29,16 @@ var _ = Describe("Test system info rest api", func() { res := get("/system_info/") var info apisv1.SystemInfoResponse Expect(decodeResponseBody(res, &info)).Should(Succeed()) - Expect(len(info.InstallID)).ShouldNot(BeEquivalentTo(0)) + Expect(len(info.PlatformID)).ShouldNot(BeEquivalentTo(0)) Expect(info.EnableCollection).Should(BeEquivalentTo(true)) - systemID := info.InstallID + systemID := info.PlatformID // check several times the systemID should not change for i := 0; i < 5; i++ { res := get("/system_info/") var checkInfo apisv1.SystemInfoResponse Expect(decodeResponseBody(res, &checkInfo)).Should(Succeed()) - Expect(checkInfo.InstallID).Should(BeEquivalentTo(systemID)) + Expect(checkInfo.PlatformID).Should(BeEquivalentTo(systemID)) } }) @@ -46,33 +46,33 @@ var _ = Describe("Test system info rest api", func() { res := get("/system_info/") var info apisv1.SystemInfoResponse Expect(decodeResponseBody(res, &info)).Should(Succeed()) - Expect(len(info.InstallID)).ShouldNot(BeEquivalentTo(0)) + Expect(len(info.PlatformID)).ShouldNot(BeEquivalentTo(0)) Expect(info.EnableCollection).Should(BeEquivalentTo(true)) - installID := info.InstallID + installID := info.PlatformID res = put("/system_info/", apisv1.SystemInfoRequest{EnableCollection: false}) info = apisv1.SystemInfoResponse{} Expect(decodeResponseBody(res, &info)).Should(Succeed()) - Expect(len(info.InstallID)).ShouldNot(BeEquivalentTo(0)) + Expect(len(info.PlatformID)).ShouldNot(BeEquivalentTo(0)) Expect(info.EnableCollection).Should(BeEquivalentTo(false)) res = get("/system_info/") var checkInfo apisv1.SystemInfoResponse Expect(decodeResponseBody(res, &checkInfo)).Should(Succeed()) - Expect(checkInfo.InstallID).Should(BeEquivalentTo(installID)) + Expect(checkInfo.PlatformID).Should(BeEquivalentTo(installID)) Expect(checkInfo.EnableCollection).Should(BeEquivalentTo(false)) res = put("/system_info/", apisv1.SystemInfoRequest{EnableCollection: true}) var enableInfo apisv1.SystemInfoResponse Expect(decodeResponseBody(res, &enableInfo)).Should(Succeed()) - Expect(len(enableInfo.InstallID)).ShouldNot(BeEquivalentTo(0)) + Expect(len(enableInfo.PlatformID)).ShouldNot(BeEquivalentTo(0)) Expect(enableInfo.EnableCollection).Should(BeEquivalentTo(true)) - Expect(enableInfo.InstallID).Should(BeEquivalentTo(installID)) + Expect(enableInfo.PlatformID).Should(BeEquivalentTo(installID)) res = get("/system_info/") var checkAgainInfo apisv1.SystemInfoResponse Expect(decodeResponseBody(res, &checkAgainInfo)).Should(Succeed()) - Expect(checkAgainInfo.InstallID).Should(BeEquivalentTo(installID)) + Expect(checkAgainInfo.PlatformID).Should(BeEquivalentTo(installID)) Expect(checkAgainInfo.EnableCollection).Should(BeEquivalentTo(true)) }) })