diff --git a/pkg/addon/cache.go b/pkg/addon/cache.go index cae3c44fd..d3c464df9 100644 --- a/pkg/addon/cache.go +++ b/pkg/addon/cache.go @@ -19,6 +19,7 @@ package addon import ( "context" "fmt" + "strings" "sync" "time" @@ -122,31 +123,27 @@ func (u *Cache) GetUIData(r Registry, addonName, version string) (*UIData, error // ListUIData will always list UIData from cache first, if not exist, read from source. func (u *Cache) ListUIData(r Registry) ([]*UIData, error) { var err error - listAddons := u.listCachedUIData(r.Name) - if listAddons != nil { - return listAddons, nil - } + var listAddons []*UIData if !IsVersionRegistry(r) { - addonMeta, err := u.ListAddonMeta(r) + listAddons = u.listCachedUIData(r.Name) + if listAddons != nil { + return listAddons, nil + } + listAddons, err = u.listUIDataAndCache(r) if err != nil { return nil, err } - listAddons, err = r.ListUIData(addonMeta, UIMetaOptions) - if err != nil { - return nil, fmt.Errorf("fail to get addons from registry %s, %w", r.Name, err) - } } else { - versionedRegistry := BuildVersionedRegistry(r.Name, r.Helm.URL, &common.HTTPOption{ - Username: r.Helm.Username, - Password: r.Helm.Password, - }) - listAddons, err = versionedRegistry.ListAddon() + listAddons = u.listVersionRegistryCachedUIData(r.Name) + if listAddons != nil { + return listAddons, nil + } + listAddons, err = u.listVersionRegistryUIDataAndCache(r) if err != nil { - log.Logger.Errorf("fail to get addons from registry %s for cache updating, %v", r.Name, err) return nil, err } } - u.putAddonUIData2Cache(r.Name, listAddons) + return listAddons, nil } @@ -181,6 +178,27 @@ func (u *Cache) listCachedUIData(name string) []*UIData { return d } +// listVersionRegistryCachedUIData will get cached addons from specified VersionRegistry in cache +func (u *Cache) listVersionRegistryCachedUIData(name string) []*UIData { + if u == nil { + return nil + } + u.mutex.RLock() + defer u.mutex.RUnlock() + d, ok := u.versionedUIData[name] + if !ok { + return nil + } + var uiDatas []*UIData + for version, uiData := range d { + if !strings.Contains(version, "-latest") { + uiDatas = append(uiDatas, uiData) + } + } + + return uiDatas +} + // getCachedAddonMeta will get cached registry meta from specified registry in cache func (u *Cache) getCachedAddonMeta(name string) map[string]SourceMeta { if u == nil { @@ -266,38 +284,54 @@ func (u *Cache) discoverAndRefreshRegistry() { for _, r := range registries { if !IsVersionRegistry(r) { - registryMeta, err := r.ListAddonMeta() + _, err = u.listUIDataAndCache(r) if err != nil { - log.Logger.Errorf("fail to list registry %s metadata, %v", r.Name, err) continue } - u.putAddonMeta2Cache(r.Name, registryMeta) - uiData, err := r.ListUIData(registryMeta, UIMetaOptions) - if err != nil { - log.Logger.Errorf("fail to get addons from registry %s for cache updating, %v", r.Name, err) - continue - } - u.putAddonUIData2Cache(r.Name, uiData) } else { - versionedRegistry := BuildVersionedRegistry(r.Name, r.Helm.URL, &common.HTTPOption{ - Username: r.Helm.Username, - Password: r.Helm.Password, - }) - uiDatas, err := versionedRegistry.ListAddon() + _, err = u.listVersionRegistryUIDataAndCache(r) if err != nil { - log.Logger.Errorf("fail to get addons from registry %s for cache updating, %v", r.Name, err) continue } - for _, addon := range uiDatas { - uiData, err := versionedRegistry.GetAddonUIData(context.Background(), addon.Name, addon.Version) - if err != nil { - log.Logger.Errorf("fail to get addon from registry %s, addon %s version %s for cache updating, %v", r.Name, addon.Name, addon.Version, err) - continue - } - u.putVersionedUIData2Cache(r.Name, addon.Name, addon.Version, uiData) - // we also no version key, if use get addonUIData without version will return this vale as latest data. - u.putVersionedUIData2Cache(r.Name, addon.Name, "latest", uiData) - } } } } + +func (u *Cache) listUIDataAndCache(r Registry) ([]*UIData, error) { + registryMeta, err := r.ListAddonMeta() + if err != nil { + log.Logger.Errorf("fail to list registry %s metadata, %v", r.Name, err) + return nil, err + } + u.putAddonMeta2Cache(r.Name, registryMeta) + uiData, err := r.ListUIData(registryMeta, UIMetaOptions) + if err != nil { + log.Logger.Errorf("fail to get addons from registry %s for cache updating, %v", r.Name, err) + return nil, err + } + u.putAddonUIData2Cache(r.Name, uiData) + return uiData, nil +} + +func (u *Cache) listVersionRegistryUIDataAndCache(r Registry) ([]*UIData, error) { + versionedRegistry := BuildVersionedRegistry(r.Name, r.Helm.URL, &common.HTTPOption{ + Username: r.Helm.Username, + Password: r.Helm.Password, + }) + uiDatas, err := versionedRegistry.ListAddon() + if err != nil { + log.Logger.Errorf("fail to get addons from registry %s for cache updating, %v", r.Name, err) + return nil, err + } + for _, addon := range uiDatas { + uiData, err := versionedRegistry.GetAddonUIData(context.Background(), addon.Name, addon.Version) + if err != nil { + log.Logger.Errorf("fail to get addon from versioned registry %s, addon %s version %s for cache updating, %v", r.Name, addon.Name, addon.Version, err) + continue + } + u.putVersionedUIData2Cache(r.Name, addon.Name, addon.Version, uiData) + // we also no version key, if use get addonUIData without version will return this vale as latest data. + u.putVersionedUIData2Cache(r.Name, addon.Name, "latest", uiData) + } + return uiDatas, nil +} diff --git a/pkg/addon/cache_test.go b/pkg/addon/cache_test.go index 4e2816b78..87cb97b73 100644 --- a/pkg/addon/cache_test.go +++ b/pkg/addon/cache_test.go @@ -52,6 +52,37 @@ func TestListCachedUIData(t *testing.T) { assert.Equal(t, u.listCachedUIData(name), addons) } +func testListUIData(t *testing.T) { + vr := Registry{Name: "helm-repo", Helm: &HelmSource{URL: "http://127.0.0.1:18083/authReg", Username: "hello", Password: "hello"}} + uiData := UIData{Meta: Meta{ + Name: "fluxcd", + Description: "Extended workload to do continuous and progressive delivery", + Icon: "https://raw.githubusercontent.com/fluxcd/flux/master/docs/_files/weave-flux.png", + Version: "1.0.0", + Tags: []string{"extended_workload", "gitops"}, + }, + AvailableVersions: []string{"1.0.0"}, + RegistryName: "helm-repo"} + addons := []*UIData{&uiData} + u := NewCache(nil) + uiDatas, err := u.ListUIData(vr) + assert.NoError(t, err) + assert.Equal(t, uiDatas, addons) +} + +func TestListVersionRegistryCachedUIData(t *testing.T) { + name := "fluxcd" + version := "v1.0.1" + uiData := &UIData{Meta: Meta{Name: name, Icon: "test.com/fluxcd.png", Version: version}} + addons := []*UIData{uiData} + vrName := "helm-repo" + u := NewCache(nil) + u.putVersionedUIData2Cache(vrName, name, version, uiData) + u.putVersionedUIData2Cache(vrName, name, "latest", uiData) + + assert.Equal(t, u.listVersionRegistryCachedUIData(vrName), addons) +} + func TestPutAddonMeta2Cache(t *testing.T) { addonMeta := map[string]SourceMeta{ "fluxcd": { diff --git a/pkg/addon/versioned_registry_test.go b/pkg/addon/versioned_registry_test.go index 0963912d8..495a1760a 100644 --- a/pkg/addon/versioned_registry_test.go +++ b/pkg/addon/versioned_registry_test.go @@ -80,6 +80,8 @@ func TestVersionRegistry(t *testing.T) { assert.NotEmpty(t, addonsInstallPackage.YAMLTemplates) assert.NotEmpty(t, addonsInstallPackage.DefSchemas) + testListUIData(t) + } var versionedHandler http.HandlerFunc = func(writer http.ResponseWriter, request *http.Request) { diff --git a/test/e2e-multicluster-test/suite_test.go b/test/e2e-multicluster-test/suite_test.go index b54ba68d1..8ce394ce7 100644 --- a/test/e2e-multicluster-test/suite_test.go +++ b/test/e2e-multicluster-test/suite_test.go @@ -79,6 +79,8 @@ var _ = AfterSuite(func() { for _, app := range apps.Items { g.Expect(k8sClient.Delete(context.Background(), app.DeepCopy())).Should(Succeed()) } + err := k8sClient.List(context.Background(), apps) + g.Expect(err, nil) g.Expect(len(apps.Items)).Should(Equal(0)) }, 3*time.Minute).Should(Succeed()) Eventually(func(g Gomega) {