Fix: change systemInfo some fields (#3715)

* add some field an calculate workflow step

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

* fix the calculate job cannot start issue

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

* fix comments

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

fix test

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

pkg/apiserver/collect/system_info_collect.go

@@ -126,7 +126,7 @@ func (i InfoCalculateCronJob) run() error {
 
 func (i InfoCalculateCronJob) calculateAndUpdate(ctx context.Context, systemInfo model.SystemInfo) error {
 
-	appCount, topKComp, topKTrait, err := i.calculateAppInfo(ctx)
+	appCount, topKComp, topKTrait, topWorkflowStep, topKPolicy, err := i.calculateAppInfo(ctx)
 	if err != nil {
 		return err
 	}
@@ -142,12 +142,14 @@ func (i InfoCalculateCronJob) calculateAndUpdate(ctx context.Context, systemInfo
 	}
 
 	statisticInfo := model.StatisticInfo{
-		AppCount:     genCountInfo(appCount),
-		TopKCompDef:  topKComp,
-		TopKTraitDef: topKTrait,
-		ClusterCount: genCountInfo(clusterCount),
-		EnabledAddon: enabledAddon,
-		UpdateTime:   time.Now(),
+		AppCount:            genCountInfo(appCount),
+		TopKCompDef:         topKComp,
+		TopKTraitDef:        topKTrait,
+		TopKWorkflowStepDef: topWorkflowStep,
+		TopKPolicyDef:       topKPolicy,
+		ClusterCount:        genClusterCountInfo(clusterCount),
+		EnabledAddon:        enabledAddon,
+		UpdateTime:          time.Now(),
 	}
 
 	systemInfo.StatisticInfo = statisticInfo
@@ -157,16 +159,18 @@ func (i InfoCalculateCronJob) calculateAndUpdate(ctx context.Context, systemInfo
 	return nil
 }
 
-func (i InfoCalculateCronJob) calculateAppInfo(ctx context.Context) (int, []string, []string, error) {
+func (i InfoCalculateCronJob) calculateAppInfo(ctx context.Context) (int, []string, []string, []string, []string, error) {
 	var err error
 	var appCount int
 	compDef := map[string]int{}
 	traitDef := map[string]int{}
+	workflowDef := map[string]int{}
+	policyDef := map[string]int{}
 
 	var app = model.Application{}
 	entities, err := i.ds.List(ctx, &app, &datastore.ListOptions{})
 	if err != nil {
-		return 0, nil, nil, err
+		return 0, nil, nil, nil, nil, err
 	}
 	for _, entity := range entities {
 		appModel, ok := entity.(*model.Application)
@@ -179,7 +183,7 @@ func (i InfoCalculateCronJob) calculateAppInfo(ctx context.Context) (int, []stri
 		}
 		comps, err := i.ds.List(ctx, &comp, &datastore.ListOptions{})
 		if err != nil {
-			return 0, nil, nil, err
+			return 0, nil, nil, nil, nil, err
 		}
 		for _, e := range comps {
 			c, ok := e.(*model.ApplicationComponent)
@@ -191,9 +195,41 @@ func (i InfoCalculateCronJob) calculateAppInfo(ctx context.Context) (int, []stri
 				traitDef[t.Type]++
 			}
 		}
+
+		workflow := model.Workflow{
+			AppPrimaryKey: app.PrimaryKey(),
+		}
+		workflows, err := i.ds.List(ctx, &workflow, &datastore.ListOptions{})
+		if err != nil {
+			return 0, nil, nil, nil, nil, err
+		}
+		for _, e := range workflows {
+			w, ok := e.(*model.Workflow)
+			if !ok {
+				continue
+			}
+			for _, step := range w.Steps {
+				workflowDef[step.Type]++
+			}
+		}
+
+		policy := model.ApplicationPolicy{
+			AppPrimaryKey: app.PrimaryKey(),
+		}
+		policies, err := i.ds.List(ctx, &policy, &datastore.ListOptions{})
+		if err != nil {
+			return 0, nil, nil, nil, nil, err
+		}
+		for _, e := range policies {
+			p, ok := e.(*model.ApplicationPolicy)
+			if !ok {
+				continue
+			}
+			policyDef[p.Type]++
+		}
 	}
 
-	return appCount, topKFrequent(compDef, TopKFrequent), topKFrequent(traitDef, TopKFrequent), nil
+	return appCount, topKFrequent(compDef, TopKFrequent), topKFrequent(traitDef, TopKFrequent), topKFrequent(workflowDef, TopKFrequent), topKFrequent(policyDef, TopKFrequent), nil
 }
 
 func (i InfoCalculateCronJob) calculateAddonInfo(ctx context.Context) (map[string]string, error) {
@@ -280,3 +316,16 @@ func genCountInfo(num int) string {
 		return ">=10000"
 	}
 }
+
+func genClusterCountInfo(num int) string {
+	switch {
+	case num < 3:
+		return "<3"
+	case num < 10:
+		return "<10"
+	case num < 50:
+		return "<50"
+	default:
+		return ">=50"
+	}
+}

pkg/apiserver/collect/system_info_collect_test.go

@@ -90,7 +90,7 @@ var _ = Describe("Test calculate cronJob", func() {
 	})
 
 	It("Test calculate app Info", func() {
-		appNum, topKCom, topKTrait, err := i.calculateAppInfo(ctx)
+		appNum, topKCom, topKTrait, _, _, err := i.calculateAppInfo(ctx)
 		Expect(err).Should(BeNil())
 		Expect(appNum).Should(BeEquivalentTo(2))
 		Expect(topKCom).Should(BeEquivalentTo([]string{"webservice", "helm"}))
@@ -131,7 +131,7 @@ var _ = Describe("Test calculate cronJob", func() {
 		Expect(ok).Should(BeTrue())
 		Expect(info.InstallID).Should(BeEquivalentTo("test-id"))
 		Expect(info.StatisticInfo.AppCount).Should(BeEquivalentTo("<10"))
-		Expect(info.StatisticInfo.ClusterCount).Should(BeEquivalentTo("<10"))
+		Expect(info.StatisticInfo.ClusterCount).Should(BeEquivalentTo("<3"))
 		Expect(info.StatisticInfo.TopKCompDef).Should(BeEquivalentTo([]string{"webservice", "helm"}))
 		Expect(info.StatisticInfo.TopKTraitDef).Should(BeEquivalentTo([]string{"rollout", "patch", "expose"}))
 		Expect(info.StatisticInfo.EnabledAddon).Should(BeEquivalentTo(map[string]string{
@@ -193,6 +193,33 @@ func TestGenCountInfo(t *testing.T) {
 	}
 }
 
+func TestGenClusterCountInfo(t *testing.T) {
+	testcases := []struct {
+		count int
+		res   string
+	}{
+		{
+			count: 2,
+			res:   "<3",
+		},
+		{
+			count: 7,
+			res:   "<10",
+		},
+		{
+			count: 34,
+			res:   "<50",
+		},
+		{
+			count: 100,
+			res:   ">=50",
+		},
+	}
+	for _, testcase := range testcases {
+		assert.Equal(t, genClusterCountInfo(testcase.count), testcase.res)
+	}
+}
+
 func TestTopKFrequent(t *testing.T) {
 	testCases := []struct {
 		def map[string]int

pkg/apiserver/model/system_info.go

@@ -51,12 +51,14 @@ type DexConfig struct {
 
 // StatisticInfo the system statistic info
 type StatisticInfo struct {
-	ClusterCount string            `json:"clusterCount,omitempty"`
-	AppCount     string            `json:"appCount,omitempty"`
-	EnabledAddon map[string]string `json:"enabledAddon,omitempty"`
-	TopKCompDef  []string          `json:"topKCompDef,omitempty"`
-	TopKTraitDef []string          `json:"topKTraitDef,omitempty"`
-	UpdateTime   time.Time         `json:"updateTime,omitempty"`
+	ClusterCount        string            `json:"clusterCount,omitempty"`
+	AppCount            string            `json:"appCount,omitempty"`
+	EnabledAddon        map[string]string `json:"enabledAddon,omitempty"`
+	TopKCompDef         []string          `json:"topKCompDef,omitempty"`
+	TopKTraitDef        []string          `json:"topKTraitDef,omitempty"`
+	TopKWorkflowStepDef []string          `json:"topKWorkflowStepDef,omitempty"`
+	TopKPolicyDef       []string          `json:"topKPolicyDef,omitempty"`
+	UpdateTime          time.Time         `json:"updateTime,omitempty"`
 }
 
 // DexStorage dex storage

pkg/apiserver/rest/apis/v1/types.go

@@ -1124,19 +1124,21 @@ type SystemInfoResponse struct {
 
 // SystemInfo system info
 type SystemInfo struct {
-	InstallID        string `json:"installID"`
+	PlatformID       string `json:"platformID"`
 	EnableCollection bool   `json:"enableCollection"`
 	LoginType        string `json:"loginType"`
 }
 
 // StatisticInfo generated by cronJob running in backend
 type StatisticInfo struct {
-	ClusterCount string            `json:"clusterCount,omitempty"`
-	AppCount     string            `json:"appCount,omitempty"`
-	EnabledAddon map[string]string `json:"enabledAddon,omitempty"`
-	TopKCompDef  []string          `json:"topKCompDef,omitempty"`
-	TopKTraitDef []string          `json:"topKTraitDef,omitempty"`
-	UpdateTime   time.Time         `json:"updateTime,omitempty"`
+	ClusterCount               string            `json:"clusterCount,omitempty"`
+	AppCount                   string            `json:"appCount,omitempty"`
+	EnableAddonList            map[string]string `json:"enableAddonList,omitempty"`
+	ComponentDefinitionTopList []string          `json:"componentDefinitionTopList,omitempty"`
+	TraitDefinitionTopList     []string          `json:"traitDefinitionTopList,omitempty"`
+	WorkflowDefinitionTopList  []string          `json:"workflowDefinitionTopList,omitempty"`
+	PolicyDefinitionTopList    []string          `json:"policyDefinitionTopList,omitempty"`
+	UpdateTime                 time.Time         `json:"updateTime,omitempty"`
 }
 
 // SystemInfoRequest request by update SystemInfo

pkg/apiserver/rest/rest_server.go

@@ -146,10 +146,11 @@ func (s *restServer) setupLeaderElection() (*leaderelection.LeaderElectionConfig
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: func(ctx context.Context) {
 				go velasync.Start(ctx, s.dataStore, restCfg, s.usecases)
-				s.runWorkflowRecordSync(ctx, s.cfg.LeaderConfig.Duration)
 				if !s.cfg.DisableStatisticCronJob {
 					collect.StartCalculatingInfoCronJob(s.dataStore)
 				}
+				// this process would block the whole process, any other handler should start before this func
+				s.runWorkflowRecordSync(ctx, s.cfg.LeaderConfig.Duration)
 			},
 			OnStoppedLeading: func() {
 				klog.Infof("leader lost: %s", s.cfg.LeaderConfig.ID)

pkg/apiserver/rest/usecase/system_info.go

@@ -101,12 +101,14 @@ func (u systemInfoUsecaseImpl) GetSystemInfo(ctx context.Context) (*v1.SystemInf
 			GitVersion:  version.GitRevision,
 		},
 		StatisticInfo: v1.StatisticInfo{
-			AppCount:     info.StatisticInfo.AppCount,
-			ClusterCount: info.StatisticInfo.ClusterCount,
-			EnabledAddon: info.StatisticInfo.EnabledAddon,
-			TopKCompDef:  info.StatisticInfo.TopKCompDef,
-			TopKTraitDef: info.StatisticInfo.TopKTraitDef,
-			UpdateTime:   info.StatisticInfo.UpdateTime,
+			AppCount:                   info.StatisticInfo.AppCount,
+			ClusterCount:               info.StatisticInfo.ClusterCount,
+			EnableAddonList:            info.StatisticInfo.EnabledAddon,
+			ComponentDefinitionTopList: info.StatisticInfo.TopKCompDef,
+			TraitDefinitionTopList:     info.StatisticInfo.TopKTraitDef,
+			WorkflowDefinitionTopList:  info.StatisticInfo.TopKWorkflowStepDef,
+			PolicyDefinitionTopList:    info.StatisticInfo.TopKPolicyDef,
+			UpdateTime:                 info.StatisticInfo.UpdateTime,
 		},
 	}, nil
 }
@@ -144,7 +146,7 @@ func (u systemInfoUsecaseImpl) UpdateSystemInfo(ctx context.Context, sysInfo v1.
 	}
 	return &v1.SystemInfoResponse{
 		SystemInfo: v1.SystemInfo{
-			InstallID:        modifiedInfo.InstallID,
+			PlatformID:       modifiedInfo.InstallID,
 			EnableCollection: modifiedInfo.EnableCollection,
 			LoginType:        modifiedInfo.LoginType,
 		},
@@ -164,7 +166,7 @@ func (u systemInfoUsecaseImpl) Init(ctx context.Context) error {
 
 func convertInfoToBase(info *model.SystemInfo) v1.SystemInfo {
 	return v1.SystemInfo{
-		InstallID:        info.InstallID,
+		PlatformID:       info.InstallID,
 		EnableCollection: info.EnableCollection,
 		LoginType:        info.LoginType,
 	}

test/e2e-apiserver-test/system_info_test.go

@@ -29,16 +29,16 @@ var _ = Describe("Test system info  rest api", func() {
 		res := get("/system_info/")
 		var info apisv1.SystemInfoResponse
 		Expect(decodeResponseBody(res, &info)).Should(Succeed())
-		Expect(len(info.InstallID)).ShouldNot(BeEquivalentTo(0))
+		Expect(len(info.PlatformID)).ShouldNot(BeEquivalentTo(0))
 		Expect(info.EnableCollection).Should(BeEquivalentTo(true))
-		systemID := info.InstallID
+		systemID := info.PlatformID
 
 		// check several times the systemID should not change
 		for i := 0; i < 5; i++ {
 			res := get("/system_info/")
 			var checkInfo apisv1.SystemInfoResponse
 			Expect(decodeResponseBody(res, &checkInfo)).Should(Succeed())
-			Expect(checkInfo.InstallID).Should(BeEquivalentTo(systemID))
+			Expect(checkInfo.PlatformID).Should(BeEquivalentTo(systemID))
 		}
 	})
 
@@ -46,33 +46,33 @@ var _ = Describe("Test system info  rest api", func() {
 		res := get("/system_info/")
 		var info apisv1.SystemInfoResponse
 		Expect(decodeResponseBody(res, &info)).Should(Succeed())
-		Expect(len(info.InstallID)).ShouldNot(BeEquivalentTo(0))
+		Expect(len(info.PlatformID)).ShouldNot(BeEquivalentTo(0))
 		Expect(info.EnableCollection).Should(BeEquivalentTo(true))
-		installID := info.InstallID
+		installID := info.PlatformID
 
 		res = put("/system_info/", apisv1.SystemInfoRequest{EnableCollection: false})
 		info = apisv1.SystemInfoResponse{}
 		Expect(decodeResponseBody(res, &info)).Should(Succeed())
-		Expect(len(info.InstallID)).ShouldNot(BeEquivalentTo(0))
+		Expect(len(info.PlatformID)).ShouldNot(BeEquivalentTo(0))
 		Expect(info.EnableCollection).Should(BeEquivalentTo(false))
 
 		res = get("/system_info/")
 		var checkInfo apisv1.SystemInfoResponse
 		Expect(decodeResponseBody(res, &checkInfo)).Should(Succeed())
-		Expect(checkInfo.InstallID).Should(BeEquivalentTo(installID))
+		Expect(checkInfo.PlatformID).Should(BeEquivalentTo(installID))
 		Expect(checkInfo.EnableCollection).Should(BeEquivalentTo(false))
 
 		res = put("/system_info/", apisv1.SystemInfoRequest{EnableCollection: true})
 		var enableInfo apisv1.SystemInfoResponse
 		Expect(decodeResponseBody(res, &enableInfo)).Should(Succeed())
-		Expect(len(enableInfo.InstallID)).ShouldNot(BeEquivalentTo(0))
+		Expect(len(enableInfo.PlatformID)).ShouldNot(BeEquivalentTo(0))
 		Expect(enableInfo.EnableCollection).Should(BeEquivalentTo(true))
-		Expect(enableInfo.InstallID).Should(BeEquivalentTo(installID))
+		Expect(enableInfo.PlatformID).Should(BeEquivalentTo(installID))
 
 		res = get("/system_info/")
 		var checkAgainInfo apisv1.SystemInfoResponse
 		Expect(decodeResponseBody(res, &checkAgainInfo)).Should(Succeed())
-		Expect(checkAgainInfo.InstallID).Should(BeEquivalentTo(installID))
+		Expect(checkAgainInfo.PlatformID).Should(BeEquivalentTo(installID))
 		Expect(checkAgainInfo.EnableCollection).Should(BeEquivalentTo(true))
 	})
 })