From 25ea10bcbfbb4cb298d21d87ef57323bd1d85bdb Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Mon, 16 Dec 2024 21:59:55 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20importer=20into=20registratio?= =?UTF-8?q?n=20(#753)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add importer into registraiton Signed-off-by: Jian Qiu * Add unit tests Signed-off-by: Jian Qiu * Add integration test Signed-off-by: Jian Qiu --------- Signed-off-by: Jian Qiu --- go.mod | 2 +- go.sum | 4 +- pkg/registration/hub/importer/importer.go | 285 +++ .../hub/importer/importer_test.go | 152 ++ .../hub/importer/options/options.go | 17 + .../hub/importer/providers/capi/provider.go | 161 ++ .../importer/providers/capi/provider_test.go | 271 +++ .../hub/importer/providers/interface.go | 65 + pkg/registration/hub/importer/renderers.go | 93 + .../hub/importer/renderers_test.go | 99 + pkg/registration/hub/manager.go | 30 + pkg/registration/hub/manager_test.go | 3 + .../registration/integration_suite_test.go | 7 + .../managedcluster_importer_test.go | 183 ++ .../capi/cluster.x-k8s.io_clusters.yaml | 1939 +++++++++++++++++ vendor/modules.txt | 2 +- .../api/feature/feature.go | 8 +- 17 files changed, 3315 insertions(+), 6 deletions(-) create mode 100644 pkg/registration/hub/importer/importer.go create mode 100644 pkg/registration/hub/importer/importer_test.go create mode 100644 pkg/registration/hub/importer/options/options.go create mode 100644 pkg/registration/hub/importer/providers/capi/provider.go create mode 100644 pkg/registration/hub/importer/providers/capi/provider_test.go create mode 100644 pkg/registration/hub/importer/providers/interface.go create mode 100644 pkg/registration/hub/importer/renderers.go create mode 100644 pkg/registration/hub/importer/renderers_test.go create mode 100644 test/integration/registration/managedcluster_importer_test.go create mode 100644 test/integration/testdeps/capi/cluster.x-k8s.io_clusters.yaml diff --git a/go.mod b/go.mod index 668a7fd5f..358e5ff84 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( k8s.io/kube-aggregator v0.31.4 k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50 - open-cluster-management.io/api v0.15.1-0.20241209025232-b62746ae96d4 + open-cluster-management.io/api v0.15.1-0.20241210025410-0ba6809d0ae2 open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f sigs.k8s.io/cluster-inventory-api v0.0.0-20240730014211-ef0154379848 sigs.k8s.io/controller-runtime v0.19.3 diff --git a/go.sum b/go.sum index f85151631..eac12db93 100644 --- a/go.sum +++ b/go.sum @@ -453,8 +453,8 @@ k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50 h1:TXRd6OdGjArh6cwlCYOqlIcyx21k81oUIYj4rmHlYx0= open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50/go.mod h1:tsBSNs9mGfVQQjXBnjgpiX6r0UM+G3iNfmzQgKhEfw4= -open-cluster-management.io/api v0.15.1-0.20241209025232-b62746ae96d4 h1:f6KU3t9s0PA6vXmAjB6A9sd52OqBqOFK2uAhk3UUBKs= -open-cluster-management.io/api v0.15.1-0.20241209025232-b62746ae96d4/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM= +open-cluster-management.io/api v0.15.1-0.20241210025410-0ba6809d0ae2 h1:zkp3VJnvexYk5fMf9/yFt6P0fQmp1WFd6Q/Y2t2jF5Q= +open-cluster-management.io/api v0.15.1-0.20241210025410-0ba6809d0ae2/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM= open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f h1:zeC7QrFNarfK2zY6jGtd+mX+yDrQQmnH/J8A7n5Nh38= open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f/go.mod h1:fi5WBsbC5K3txKb8eRLuP0Sim/Oqz/PHX18skAEyjiA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY= diff --git a/pkg/registration/hub/importer/importer.go b/pkg/registration/hub/importer/importer.go new file mode 100644 index 000000000..a8fd69b65 --- /dev/null +++ b/pkg/registration/hub/importer/importer.go @@ -0,0 +1,285 @@ +package importer + +import ( + "context" + "fmt" + + "github.com/openshift/api" + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/resource/resourceapply" + "github.com/openshift/library-go/pkg/operator/resource/resourcehelper" + "github.com/openshift/library-go/pkg/operator/resource/resourcemerge" + appsv1 "k8s.io/api/apps/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + + clusterclientset "open-cluster-management.io/api/client/cluster/clientset/versioned" + clusterinformerv1 "open-cluster-management.io/api/client/cluster/informers/externalversions/cluster/v1" + clusterlisterv1 "open-cluster-management.io/api/client/cluster/listers/cluster/v1" + operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned" + v1 "open-cluster-management.io/api/cluster/v1" + operatorv1 "open-cluster-management.io/api/operator/v1" + "open-cluster-management.io/sdk-go/pkg/patcher" + + "open-cluster-management.io/ocm/pkg/common/queue" + "open-cluster-management.io/ocm/pkg/operator/helpers/chart" + cloudproviders "open-cluster-management.io/ocm/pkg/registration/hub/importer/providers" +) + +const ( + operatorNamesapce = "open-cluster-management" + bootstrapSA = "cluster-bootstrap" + ManagedClusterConditionImported = "Imported" +) + +var ( + genericScheme = runtime.NewScheme() + genericCodecs = serializer.NewCodecFactory(genericScheme) + genericCodec = genericCodecs.UniversalDeserializer() +) + +func init() { + utilruntime.Must(api.InstallKube(genericScheme)) + utilruntime.Must(apiextensionsv1.AddToScheme(genericScheme)) + utilruntime.Must(operatorv1.Install(genericScheme)) +} + +// KlusterletConfigRenderer renders the config for klusterlet chart. +type KlusterletConfigRenderer func( + ctx context.Context, config *chart.KlusterletChartConfig) (*chart.KlusterletChartConfig, error) + +type Importer struct { + providers []cloudproviders.Interface + clusterClient clusterclientset.Interface + clusterLister clusterlisterv1.ManagedClusterLister + renders []KlusterletConfigRenderer + patcher patcher.Patcher[*v1.ManagedCluster, v1.ManagedClusterSpec, v1.ManagedClusterStatus] +} + +// NewImporter creates an auto import controller +func NewImporter( + renders []KlusterletConfigRenderer, + clusterClient clusterclientset.Interface, + clusterInformer clusterinformerv1.ManagedClusterInformer, + providers []cloudproviders.Interface, + recorder events.Recorder) factory.Controller { + controllerName := "managed-cluster-importer" + syncCtx := factory.NewSyncContext(controllerName, recorder) + + i := &Importer{ + providers: providers, + clusterClient: clusterClient, + clusterLister: clusterInformer.Lister(), + renders: renders, + patcher: patcher.NewPatcher[ + *v1.ManagedCluster, v1.ManagedClusterSpec, v1.ManagedClusterStatus]( + clusterClient.ClusterV1().ManagedClusters()), + } + + for _, provider := range providers { + provider.Register(syncCtx) + } + + return factory.New().WithInformersQueueKeysFunc(queue.QueueKeyByMetaName, clusterInformer.Informer()). + WithSyncContext(syncCtx).WithSync(i.sync).ToController(controllerName, recorder) +} + +func (i *Importer) sync(ctx context.Context, syncCtx factory.SyncContext) error { + clusterName := syncCtx.QueueKey() + logger := klog.FromContext(ctx) + logger.V(4).Info("Reconciling key", "clusterName", clusterName) + + cluster, err := i.clusterLister.Get(clusterName) + switch { + case errors.IsNotFound(err): + return nil + case err != nil: + return err + } + + // If the cluster is imported, skip the reconcile + if meta.IsStatusConditionTrue(cluster.Status.Conditions, ManagedClusterConditionImported) { + return nil + } + + // get provider from the provider list + var provider cloudproviders.Interface + for _, p := range i.providers { + if p.IsManagedClusterOwner(cluster) { + provider = p + break + } + } + if provider == nil { + logger.V(2).Info("provider not found for cluster", "cluster", cluster.Name) + return nil + } + + newCluster := cluster.DeepCopy() + newCluster, err = i.reconcile(ctx, logger, syncCtx.Recorder(), provider, newCluster) + updated, updatedErr := i.patcher.PatchStatus(ctx, newCluster, newCluster.Status, cluster.Status) + if updatedErr != nil { + return updatedErr + } + if err != nil { + return err + } + if updated { + syncCtx.Recorder().Eventf( + "ManagedClusterImported", "managed cluster %s is imported", clusterName) + } + + return nil +} + +func (i *Importer) reconcile( + ctx context.Context, + logger klog.Logger, + recorder events.Recorder, + provider cloudproviders.Interface, + cluster *v1.ManagedCluster) (*v1.ManagedCluster, error) { + clients, err := provider.Clients(ctx, cluster) + if err != nil { + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: ManagedClusterConditionImported, + Status: metav1.ConditionFalse, + Reason: "KubeConfigGetFailed", + Message: fmt.Sprintf("failed to get kubeconfig. See errors:\n%s", + err.Error()), + }) + return cluster, err + } + + if clients == nil { + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: ManagedClusterConditionImported, + Status: metav1.ConditionFalse, + Reason: "KubeConfigNotFound", + Message: "Secret for kubeconfig is not found.", + }) + return cluster, nil + } + + // render the klsuterlet chart config + klusterletChartConfig := &chart.KlusterletChartConfig{ + CreateNamespace: true, + Klusterlet: chart.KlusterletConfig{ + Create: true, + ClusterName: cluster.Name, + ResourceRequirement: operatorv1.ResourceRequirement{ + Type: operatorv1.ResourceQosClassDefault, + }, + }, + } + for _, renderer := range i.renders { + klusterletChartConfig, err = renderer(ctx, klusterletChartConfig) + if err != nil { + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: ManagedClusterConditionImported, + Status: metav1.ConditionFalse, + Reason: "ConfigRendererFailed", + Message: fmt.Sprintf("failed to render config. See errors:\n%s", + err.Error()), + }) + return cluster, err + } + } + rawManifests, err := chart.RenderKlusterletChart(klusterletChartConfig, operatorNamesapce) + if err != nil { + return cluster, err + } + + clientHolder := resourceapply.NewKubeClientHolder(clients.KubeClient). + WithAPIExtensionsClient(clients.APIExtClient).WithDynamicClient(clients.DynamicClient) + cache := resourceapply.NewResourceCache() + var results []resourceapply.ApplyResult + for _, manifest := range rawManifests { + requiredObj, _, err := genericCodec.Decode(manifest, nil, nil) + if err != nil { + logger.Error(err, "failed to decode manifest", "manifest", manifest) + return cluster, err + } + result := resourceapply.ApplyResult{} + switch t := requiredObj.(type) { + case *appsv1.Deployment: + result.Result, result.Changed, result.Error = resourceapply.ApplyDeployment( + ctx, clients.KubeClient.AppsV1(), recorder, t, 0) + results = append(results, result) + case *operatorv1.Klusterlet: + result.Result, result.Changed, result.Error = ApplyKlusterlet( + ctx, clients.OperatorClient, recorder, t) + results = append(results, result) + default: + tempResults := resourceapply.ApplyDirectly(ctx, clientHolder, recorder, cache, + func(name string) ([]byte, error) { + return manifest, nil + }, + "manifest") + results = append(results, tempResults...) + } + } + + var errs []error + for _, result := range results { + if result.Error != nil { + errs = append(errs, result.Error) + } + } + if len(errs) > 0 { + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: ManagedClusterConditionImported, + Status: metav1.ConditionFalse, + Reason: "ImportFailed", + Message: fmt.Sprintf("failed to import the klusterlet. See errors:\n%s", + utilerrors.NewAggregate(errs).Error()), + }) + } else { + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: ManagedClusterConditionImported, + Status: metav1.ConditionTrue, + Reason: "ImportSucceed", + }) + } + + return cluster, utilerrors.NewAggregate(errs) +} + +func ApplyKlusterlet( + ctx context.Context, + client operatorclient.Interface, + recorder events.Recorder, + required *operatorv1.Klusterlet) (*operatorv1.Klusterlet, bool, error) { + existing, err := client.OperatorV1().Klusterlets().Get(ctx, required.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + requiredCopy := required.DeepCopy() + actual, err := client.OperatorV1().Klusterlets().Create(ctx, requiredCopy, metav1.CreateOptions{}) + resourcehelper.ReportCreateEvent(recorder, required, err) + return actual, true, err + } + if err != nil { + return nil, false, err + } + + modified := pointer.Bool(false) + existingCopy := existing.DeepCopy() + resourcemerge.EnsureObjectMeta(modified, &existingCopy.ObjectMeta, required.ObjectMeta) + + if !*modified && equality.Semantic.DeepEqual(existingCopy.Spec, required.Spec) { + return existingCopy, false, nil + } + + existingCopy.Spec = required.Spec + actual, err := client.OperatorV1().Klusterlets().Update(ctx, existingCopy, metav1.UpdateOptions{}) + resourcehelper.ReportUpdateEvent(recorder, required, err) + return actual, true, err +} diff --git a/pkg/registration/hub/importer/importer_test.go b/pkg/registration/hub/importer/importer_test.go new file mode 100644 index 000000000..bc770e1a2 --- /dev/null +++ b/pkg/registration/hub/importer/importer_test.go @@ -0,0 +1,152 @@ +package importer + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/openshift/library-go/pkg/controller/factory" + fakeapiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + fakedynamic "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + + fakeclusterclient "open-cluster-management.io/api/client/cluster/clientset/versioned/fake" + clusterinformers "open-cluster-management.io/api/client/cluster/informers/externalversions" + fakeoperatorclient "open-cluster-management.io/api/client/operator/clientset/versioned/fake" + clusterv1 "open-cluster-management.io/api/cluster/v1" + "open-cluster-management.io/sdk-go/pkg/patcher" + + testingcommon "open-cluster-management.io/ocm/pkg/common/testing" + "open-cluster-management.io/ocm/pkg/registration/hub/importer/providers" + cloudproviders "open-cluster-management.io/ocm/pkg/registration/hub/importer/providers" +) + +func TestSync(t *testing.T) { + cases := []struct { + name string + provider *fakeProvider + key string + cluster *clusterv1.ManagedCluster + validate func(t *testing.T, actions []clienttesting.Action) + }{ + { + name: "import succeed", + provider: &fakeProvider{isOwned: true}, + key: "cluster1", + cluster: &clusterv1.ManagedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster1"}}, + validate: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertActions(t, actions, "patch") + patch := actions[0].(clienttesting.PatchAction).GetPatch() + managedCluster := &clusterv1.ManagedCluster{} + err := json.Unmarshal(patch, managedCluster) + if err != nil { + t.Fatal(err) + } + if !meta.IsStatusConditionTrue(managedCluster.Status.Conditions, ManagedClusterConditionImported) { + t.Errorf("expected managed cluster to be imported") + } + }, + }, + { + name: "no cluster", + provider: &fakeProvider{isOwned: true}, + key: "cluster1", + cluster: &clusterv1.ManagedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster2"}}, + validate: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertNoActions(t, actions) + }, + }, + { + name: "not owned by the provider", + provider: &fakeProvider{isOwned: false}, + key: "cluster1", + cluster: &clusterv1.ManagedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster1"}}, + validate: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertNoActions(t, actions) + }, + }, + { + name: "clients for remote cluster is not generated", + provider: &fakeProvider{isOwned: true, noClients: true}, + key: "cluster1", + cluster: &clusterv1.ManagedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster1"}}, + validate: func(t *testing.T, actions []clienttesting.Action) { + testingcommon.AssertActions(t, actions, "patch") + patch := actions[0].(clienttesting.PatchAction).GetPatch() + managedCluster := &clusterv1.ManagedCluster{} + err := json.Unmarshal(patch, managedCluster) + if err != nil { + t.Fatal(err) + } + if !meta.IsStatusConditionFalse(managedCluster.Status.Conditions, ManagedClusterConditionImported) { + t.Errorf("expected managed cluster to be imported") + } + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + clusterClient := fakeclusterclient.NewSimpleClientset(c.cluster) + clusterInformer := clusterinformers.NewSharedInformerFactory( + clusterClient, 10*time.Minute).Cluster().V1().ManagedClusters() + clusterStore := clusterInformer.Informer().GetStore() + if err := clusterStore.Add(c.cluster); err != nil { + t.Fatal(err) + } + importer := &Importer{ + providers: []cloudproviders.Interface{c.provider}, + clusterClient: clusterClient, + clusterLister: clusterInformer.Lister(), + patcher: patcher.NewPatcher[ + *clusterv1.ManagedCluster, clusterv1.ManagedClusterSpec, clusterv1.ManagedClusterStatus]( + clusterClient.ClusterV1().ManagedClusters()), + } + err := importer.sync(context.TODO(), testingcommon.NewFakeSyncContext(t, c.key)) + if err != nil { + t.Fatal(err) + } + c.validate(t, clusterClient.Actions()) + }) + } +} + +type fakeProvider struct { + isOwned bool + noClients bool + kubeConfigErr error +} + +// KubeConfig is to return the config to connect to the target cluster. +func (f *fakeProvider) Clients(_ context.Context, _ *clusterv1.ManagedCluster) (*providers.Clients, error) { + if f.kubeConfigErr != nil { + return nil, f.kubeConfigErr + } + if f.noClients { + return nil, nil + } + return &providers.Clients{ + KubeClient: kubefake.NewClientset(), + // due to https://github.com/kubernetes/kubernetes/issues/126850, still need to use NewSimpleClientset + APIExtClient: fakeapiextensions.NewSimpleClientset(), + OperatorClient: fakeoperatorclient.NewSimpleClientset(), + DynamicClient: fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), + }, nil +} + +// IsManagedClusterOwner check if the provider is used to manage this cluster +func (f *fakeProvider) IsManagedClusterOwner(_ *clusterv1.ManagedCluster) bool { + return f.isOwned +} + +// Register registers the provider to the importer. The provider should enqueue the resource +// into the queue with the name of the managed cluster +func (f *fakeProvider) Register(_ factory.SyncContext) {} + +// Run starts the provider +func (f *fakeProvider) Run(_ context.Context) {} diff --git a/pkg/registration/hub/importer/options/options.go b/pkg/registration/hub/importer/options/options.go new file mode 100644 index 000000000..3d480fec5 --- /dev/null +++ b/pkg/registration/hub/importer/options/options.go @@ -0,0 +1,17 @@ +package options + +import "github.com/spf13/pflag" + +type Options struct { + APIServerURL string +} + +func New() *Options { + return &Options{} +} + +// AddFlags registers flags for manager +func (m *Options) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&m.APIServerURL, "hub-apiserver-url", m.APIServerURL, + "APIServer URL of the hub cluster that the spoke cluster can access, Only used for spoke cluster import") +} diff --git a/pkg/registration/hub/importer/providers/capi/provider.go b/pkg/registration/hub/importer/providers/capi/provider.go new file mode 100644 index 000000000..dcf0f3fa8 --- /dev/null +++ b/pkg/registration/hub/importer/providers/capi/provider.go @@ -0,0 +1,161 @@ +package capi + +import ( + "context" + "fmt" + "time" + + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + clusterinformerv1 "open-cluster-management.io/api/client/cluster/informers/externalversions/cluster/v1" + clusterv1 "open-cluster-management.io/api/cluster/v1" + + "open-cluster-management.io/ocm/pkg/registration/hub/importer/providers" +) + +var ClusterAPIGVR = schema.GroupVersionResource{ + Group: "cluster.x-k8s.io", + Version: "v1beta1", + Resource: "clusters", +} + +const ( + ByCAPIResource = "by-capi-resource" + CAPIAnnotationKey = "cluster.x-k8s.io/cluster" +) + +type CAPIProvider struct { + informer dynamicinformer.DynamicSharedInformerFactory + lister cache.GenericLister + kubeClient kubernetes.Interface + managedClusterIndexer cache.Indexer +} + +func NewCAPIProvider( + kubeconfig *rest.Config, clusterInformer clusterinformerv1.ManagedClusterInformer) providers.Interface { + dynamicClient := dynamic.NewForConfigOrDie(kubeconfig) + kubeClient := kubernetes.NewForConfigOrDie(kubeconfig) + + dynamicInformer := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 30*time.Minute) + + utilruntime.Must(clusterInformer.Informer().AddIndexers(cache.Indexers{ + ByCAPIResource: indexByCAPIResource, + })) + + return &CAPIProvider{ + informer: dynamicInformer, + lister: dynamicInformer.ForResource(ClusterAPIGVR).Lister(), + kubeClient: kubeClient, + managedClusterIndexer: clusterInformer.Informer().GetIndexer(), + } +} + +func (c *CAPIProvider) Clients(ctx context.Context, cluster *clusterv1.ManagedCluster) (*providers.Clients, error) { + logger := klog.FromContext(ctx) + clusterKey := capiNameFromManagedCluster(cluster) + namespace, name, err := cache.SplitMetaNamespaceKey(clusterKey) + if err != nil { + return nil, err + } + _, err = c.lister.ByNamespace(namespace).Get(name) + switch { + case apierrors.IsNotFound(err): + logger.V(4).Info("cluster is not found", "name", name, "namespace", namespace) + // TODO(qiujian16) need to consider requeue in this case, since secrets is not watched. + return nil, nil + case err != nil: + return nil, err + } + + secret, err := c.kubeClient.CoreV1().Secrets(namespace).Get(ctx, name+"-kubeconfig", metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + logger.V(4).Info( + "kubeconfig secret is not found", "name", name+"-kubeconfig", "namespace", namespace) + return nil, nil + case err != nil: + return nil, err + } + + data, ok := secret.Data["value"] + if !ok { + return nil, errors.Errorf("missing key %q in secret data", name) + } + + configOverride, err := clientcmd.NewClientConfigFromBytes(data) + if err != nil { + return nil, err + } + + config, err := configOverride.ClientConfig() + if err != nil { + return nil, err + } + return providers.NewClient(config) +} + +func (c *CAPIProvider) Register(syncCtx factory.SyncContext) { + _, err := c.informer.ForResource(ClusterAPIGVR).Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.enqueueManagedClusterByCAPI(obj, syncCtx) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + c.enqueueManagedClusterByCAPI(newObj, syncCtx) + }, + }) + utilruntime.HandleError(err) +} + +func (c *CAPIProvider) IsManagedClusterOwner(cluster *clusterv1.ManagedCluster) bool { + clusterKey := capiNameFromManagedCluster(cluster) + namespace, name, _ := cache.SplitMetaNamespaceKey(clusterKey) + _, err := c.lister.ByNamespace(namespace).Get(name) + return err == nil +} + +func (c *CAPIProvider) Run(ctx context.Context) { + c.informer.Start(ctx.Done()) +} + +func (c *CAPIProvider) enqueueManagedClusterByCAPI(obj interface{}, syncCtx factory.SyncContext) { + accessor, _ := meta.Accessor(obj) + objs, err := c.managedClusterIndexer.ByIndex(ByCAPIResource, fmt.Sprintf( + "%s/%s", accessor.GetNamespace(), accessor.GetName())) + if err != nil { + return + } + for _, obj := range objs { + accessor, _ := meta.Accessor(obj) + syncCtx.Queue().Add(accessor.GetName()) + } +} + +func indexByCAPIResource(obj interface{}) ([]string, error) { + cluster, ok := obj.(*clusterv1.ManagedCluster) + if !ok { + return []string{}, nil + } + return []string{capiNameFromManagedCluster(cluster)}, nil +} + +func capiNameFromManagedCluster(cluster *clusterv1.ManagedCluster) string { + if len(cluster.Annotations) > 0 { + if key, ok := cluster.Annotations[CAPIAnnotationKey]; ok { + return key + } + } + return fmt.Sprintf("%s/%s", cluster.Name, cluster.Name) +} diff --git a/pkg/registration/hub/importer/providers/capi/provider_test.go b/pkg/registration/hub/importer/providers/capi/provider_test.go new file mode 100644 index 000000000..d8d10722d --- /dev/null +++ b/pkg/registration/hub/importer/providers/capi/provider_test.go @@ -0,0 +1,271 @@ +package capi + +import ( + "context" + "testing" + + "github.com/ghodss/yaml" + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/library-go/pkg/operator/events/eventstesting" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic/dynamicinformer" + fakedynamic "k8s.io/client-go/dynamic/fake" + fakekube "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" + + fakecluster "open-cluster-management.io/api/client/cluster/clientset/versioned/fake" + clusterinformers "open-cluster-management.io/api/client/cluster/informers/externalversions" + clusterv1 "open-cluster-management.io/api/cluster/v1" + + testingcommon "open-cluster-management.io/ocm/pkg/common/testing" +) + +func TestEnqueu(t *testing.T) { + cases := []struct { + name string + cluster *clusterv1.ManagedCluster + capiName string + capiNamespace string + expectedKey string + }{ + { + name: "enqueu by name", + cluster: &clusterv1.ManagedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster1"}}, + capiName: "cluster1", + capiNamespace: "cluster1", + expectedKey: "cluster1", + }, + { + name: "enqueu by annotation", + cluster: &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster2", + Annotations: map[string]string{ + CAPIAnnotationKey: "capi/cluster1", + }, + }, + }, + capiName: "cluster1", + capiNamespace: "capi", + expectedKey: "cluster2", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + client := fakecluster.NewSimpleClientset(c.cluster) + informerFactory := clusterinformers.NewSharedInformerFactory(client, 0) + clusterInformer := informerFactory.Cluster().V1().ManagedClusters() + if err := clusterInformer.Informer().AddIndexers(cache.Indexers{ + ByCAPIResource: indexByCAPIResource, + }); err != nil { + t.Fatal(err) + } + if err := clusterInformer.Informer().GetStore().Add(c.cluster); err != nil { + t.Fatal(err) + } + + provider := &CAPIProvider{ + managedClusterIndexer: clusterInformer.Informer().GetIndexer(), + } + syncCtx := factory.NewSyncContext("test", eventstesting.NewTestingEventRecorder(t)) + provider.enqueueManagedClusterByCAPI(&metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.capiName, + Namespace: c.capiNamespace, + }, + }, syncCtx) + if i, _ := syncCtx.Queue().Get(); i.(string) != c.expectedKey { + t.Errorf("expected key %s but got %s", c.expectedKey, syncCtx.QueueKey()) + } + }) + } +} + +func TestClients(t *testing.T) { + cases := []struct { + name string + capiObjects []runtime.Object + kubeObjects []runtime.Object + cluster *clusterv1.ManagedCluster + expectErr bool + }{ + { + name: "capi cluster not found", + cluster: &clusterv1.ManagedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster1"}}, + }, + { + name: "secret not found", + cluster: &clusterv1.ManagedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster1"}}, + capiObjects: []runtime.Object{ + testingcommon.NewUnstructured( + "cluster.x-k8s.io/v1beta1", "Cluster", "cluster1", "cluster1")}, + }, + { + name: "secret found with invalid key", + cluster: &clusterv1.ManagedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster1"}}, + capiObjects: []runtime.Object{ + testingcommon.NewUnstructured( + "cluster.x-k8s.io/v1beta1", "Cluster", "cluster1", "cluster1")}, + kubeObjects: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1-kubeconfig", + Namespace: "cluster1", + }, + }, + }, + expectErr: true, + }, + { + name: "build client successfully", + cluster: &clusterv1.ManagedCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster1"}}, + capiObjects: []runtime.Object{ + testingcommon.NewUnstructured( + "cluster.x-k8s.io/v1beta1", "Cluster", "cluster1", "cluster1")}, + kubeObjects: []runtime.Object{ + func() *corev1.Secret { + clientConfig := clientcmdapiv1.Config{ + // Define a cluster stanza based on the bootstrap kubeconfig. + Clusters: []clientcmdapiv1.NamedCluster{ + { + Name: "hub", + Cluster: clientcmdapiv1.Cluster{ + Server: "https://test", + }, + }, + }, + // Define auth based on the obtained client cert. + AuthInfos: []clientcmdapiv1.NamedAuthInfo{ + { + Name: "bootstrap", + AuthInfo: clientcmdapiv1.AuthInfo{ + Token: "test", + }, + }, + }, + // Define a context that connects the auth info and cluster, and set it as the default + Contexts: []clientcmdapiv1.NamedContext{ + { + Name: "bootstrap", + Context: clientcmdapiv1.Context{ + Cluster: "hub", + AuthInfo: "bootstrap", + Namespace: "default", + }, + }, + }, + CurrentContext: "bootstrap", + } + bootstrapConfigBytes, _ := yaml.Marshal(clientConfig) + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1-kubeconfig", + Namespace: "cluster1", + }, + Data: map[string][]byte{ + "value": bootstrapConfigBytes, + }, + } + }(), + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + dynamicClient := fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), c.capiObjects...) + kubeClient := fakekube.NewClientset(c.kubeObjects...) + dynamicInformers := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0) + for _, capiObj := range c.capiObjects { + if err := dynamicInformers.ForResource(ClusterAPIGVR).Informer().GetStore().Add(capiObj); err != nil { + t.Fatal(err) + } + } + provider := &CAPIProvider{ + kubeClient: kubeClient, + informer: dynamicInformers, + lister: dynamicInformers.ForResource(ClusterAPIGVR).Lister(), + } + _, err := provider.Clients(context.TODO(), c.cluster) + if c.expectErr && err == nil { + t.Errorf("expected error but got nil") + } + if !c.expectErr && err != nil { + t.Errorf("expected no error but got %v", err) + } + }) + } +} + +func TestIsManagedClusterOwner(t *testing.T) { + cases := []struct { + name string + capiObjects []runtime.Object + cluster *clusterv1.ManagedCluster + expectedOwn bool + }{ + { + name: "by cluster name", + capiObjects: []runtime.Object{ + testingcommon.NewUnstructured( + "cluster.x-k8s.io/v1beta1", "Cluster", "cluster1", "cluster1")}, + cluster: &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster1"}, + }, + expectedOwn: true, + }, + { + name: "by cluster annotation", + capiObjects: []runtime.Object{ + testingcommon.NewUnstructured( + "cluster.x-k8s.io/v1beta1", "Cluster", "capi", "cluster1")}, + cluster: &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster2", + Annotations: map[string]string{ + CAPIAnnotationKey: "capi/cluster1", + }, + }, + }, + expectedOwn: true, + }, + { + name: "by cluster annotation", + capiObjects: []runtime.Object{ + testingcommon.NewUnstructured( + "cluster.x-k8s.io/v1beta1", "Cluster", "capi", "cluster2")}, + cluster: &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster2", + Annotations: map[string]string{ + CAPIAnnotationKey: "capi/cluster1", + }, + }, + }, + expectedOwn: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + dynamicClient := fakedynamic.NewSimpleDynamicClient(runtime.NewScheme(), c.capiObjects...) + dynamicInformers := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0) + for _, capiObj := range c.capiObjects { + if err := dynamicInformers.ForResource(ClusterAPIGVR).Informer().GetStore().Add(capiObj); err != nil { + t.Fatal(err) + } + } + provider := &CAPIProvider{ + lister: dynamicInformers.ForResource(ClusterAPIGVR).Lister(), + } + owned := provider.IsManagedClusterOwner(c.cluster) + if c.expectedOwn != owned { + t.Errorf("expected owned cluster %t but got %t", c.expectedOwn, owned) + } + }) + } +} diff --git a/pkg/registration/hub/importer/providers/interface.go b/pkg/registration/hub/importer/providers/interface.go new file mode 100644 index 000000000..4d4d950b7 --- /dev/null +++ b/pkg/registration/hub/importer/providers/interface.go @@ -0,0 +1,65 @@ +package providers + +import ( + "context" + + "github.com/openshift/library-go/pkg/controller/factory" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned" + clusterv1 "open-cluster-management.io/api/cluster/v1" +) + +// Interface is the interface that a cluster provider should implement +type Interface interface { + // Clients returns the client to connect to the target cluster. The client should have the sufficient + // permission to create CRDs/operator and klusterlet CR in the remote cluster. + Clients(ctx context.Context, cluster *clusterv1.ManagedCluster) (*Clients, error) + + // IsManagedClusterOwner check if the provider is used to manage this cluster + IsManagedClusterOwner(cluster *clusterv1.ManagedCluster) bool + + // Register registers the provider to the importer. The provider should enqueue the resource + // into the queue with the name of the managed cluster + Register(syncCtx factory.SyncContext) + + // Run starts the provider. The provider might need to watch the provider related resources + // on the hub cluster, or start a periodic task. + Run(ctx context.Context) +} + +type Clients struct { + KubeClient kubernetes.Interface + APIExtClient apiextensionsclient.Interface + OperatorClient operatorclient.Interface + DynamicClient dynamic.Interface +} + +func NewClient(config *rest.Config) (*Clients, error) { + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + hubApiExtensionClient, err := apiextensionsclient.NewForConfig(config) + if err != nil { + return nil, err + } + operatorClient, err := operatorclient.NewForConfig(config) + if err != nil { + return nil, err + } + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, err + } + + return &Clients{ + APIExtClient: hubApiExtensionClient, + KubeClient: kubeClient, + OperatorClient: operatorClient, + DynamicClient: dynamicClient, + }, nil +} diff --git a/pkg/registration/hub/importer/renderers.go b/pkg/registration/hub/importer/renderers.go new file mode 100644 index 000000000..dbbbedb88 --- /dev/null +++ b/pkg/registration/hub/importer/renderers.go @@ -0,0 +1,93 @@ +package importer + +import ( + "context" + "fmt" + + "github.com/ghodss/yaml" + authv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" + "k8s.io/utils/ptr" + + sdkhelpers "open-cluster-management.io/sdk-go/pkg/helpers" + + "open-cluster-management.io/ocm/pkg/operator/helpers/chart" +) + +func RenderBootstrapHubKubeConfig( + kubeClient kubernetes.Interface, apiServerURL string) KlusterletConfigRenderer { + return func(ctx context.Context, config *chart.KlusterletChartConfig) (*chart.KlusterletChartConfig, error) { + // get bootstrap token + tr, err := kubeClient.CoreV1(). + ServiceAccounts(operatorNamesapce). + CreateToken(ctx, bootstrapSA, &authv1.TokenRequest{ + Spec: authv1.TokenRequestSpec{ + // token expired in 1 hour + ExpirationSeconds: ptr.To[int64](3600), + }, + }, metav1.CreateOptions{}) + if err != nil { + return config, fmt.Errorf( + "failed to get token from sa %s/%s: %v", operatorNamesapce, bootstrapSA, err) + } + + // get apisever url + url := apiServerURL + if len(url) == 0 { + url, err = sdkhelpers.GetAPIServer(kubeClient) + if err != nil { + return config, err + } + } + + // get cabundle + ca, err := sdkhelpers.GetCACert(kubeClient) + if err != nil { + return config, err + } + + clientConfig := clientcmdapiv1.Config{ + // Define a cluster stanza based on the bootstrap kubeconfig. + Clusters: []clientcmdapiv1.NamedCluster{ + { + Name: "hub", + Cluster: clientcmdapiv1.Cluster{ + Server: url, + CertificateAuthorityData: ca, + }, + }, + }, + // Define auth based on the obtained client cert. + AuthInfos: []clientcmdapiv1.NamedAuthInfo{ + { + Name: "bootstrap", + AuthInfo: clientcmdapiv1.AuthInfo{ + Token: tr.Status.Token, + }, + }, + }, + // Define a context that connects the auth info and cluster, and set it as the default + Contexts: []clientcmdapiv1.NamedContext{ + { + Name: "bootstrap", + Context: clientcmdapiv1.Context{ + Cluster: "hub", + AuthInfo: "bootstrap", + Namespace: "default", + }, + }, + }, + CurrentContext: "bootstrap", + } + + bootstrapConfigBytes, err := yaml.Marshal(clientConfig) + if err != nil { + return config, err + } + + config.BootstrapHubKubeConfig = string(bootstrapConfigBytes) + return config, nil + } +} diff --git a/pkg/registration/hub/importer/renderers_test.go b/pkg/registration/hub/importer/renderers_test.go new file mode 100644 index 000000000..23ae084a5 --- /dev/null +++ b/pkg/registration/hub/importer/renderers_test.go @@ -0,0 +1,99 @@ +package importer + +import ( + "context" + "testing" + + "github.com/ghodss/yaml" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/clientcmd" + clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" + + "open-cluster-management.io/ocm/pkg/operator/helpers/chart" +) + +func TestRenderBootstrapHubKubeConfig(t *testing.T) { + cases := []struct { + name string + objects []runtime.Object + apiserverURL string + expectedURL string + }{ + { + name: "render apiserver from input", + apiserverURL: "https://127.0.0.1:6443", + expectedURL: "https://127.0.0.1:6443", + }, + { + name: "render apiserver from cluster-info", + objects: []runtime.Object{ + func() *corev1.ConfigMap { + config := clientcmdapiv1.Config{ + // Define a cluster stanza based on the bootstrap kubeconfig. + Clusters: []clientcmdapiv1.NamedCluster{ + { + Name: "hub", + Cluster: clientcmdapiv1.Cluster{ + Server: "https://test", + }, + }, + }, + } + bootstrapConfigBytes, _ := yaml.Marshal(config) + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-info", + Namespace: "kube-public", + }, + Data: map[string]string{ + "kubeconfig": string(bootstrapConfigBytes), + }, + } + }(), + }, + expectedURL: "https://test", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + client := kubefake.NewClientset(c.objects...) + client.PrependReactor("create", "serviceaccounts/token", + func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) { + act, ok := action.(clienttesting.CreateActionImpl) + if !ok { + return false, nil, nil + } + tokenReq, ok := act.Object.(*authenticationv1.TokenRequest) + if !ok { + return false, nil, nil + } + tokenReq.Status.Token = "token" + return true, tokenReq, nil + }, + ) + config := &chart.KlusterletChartConfig{} + config, err := RenderBootstrapHubKubeConfig(client, c.apiserverURL)(context.TODO(), config) + if err != nil { + t.Fatalf("failed to render bootstrap hub kubeconfig: %v", err) + } + kConfig, err := clientcmd.NewClientConfigFromBytes([]byte(config.BootstrapHubKubeConfig)) + if err != nil { + t.Fatalf("failed to load bootstrap hub kubeconfig: %v", err) + } + rawConfig, err := kConfig.RawConfig() + if err != nil { + t.Fatalf("failed to load bootstrap hub kubeconfig: %v", err) + } + cluster := rawConfig.Contexts[rawConfig.CurrentContext].Cluster + if rawConfig.Clusters[cluster].Server != c.expectedURL { + t.Errorf("apiserver is not rendered correctly") + } + }) + } +} diff --git a/pkg/registration/hub/manager.go b/pkg/registration/hub/manager.go index 8b5782319..8bc303dac 100644 --- a/pkg/registration/hub/manager.go +++ b/pkg/registration/hub/manager.go @@ -31,6 +31,10 @@ import ( "open-cluster-management.io/ocm/pkg/registration/hub/clusterprofile" "open-cluster-management.io/ocm/pkg/registration/hub/clusterrole" "open-cluster-management.io/ocm/pkg/registration/hub/gc" + "open-cluster-management.io/ocm/pkg/registration/hub/importer" + importeroptions "open-cluster-management.io/ocm/pkg/registration/hub/importer/options" + cloudproviders "open-cluster-management.io/ocm/pkg/registration/hub/importer/providers" + "open-cluster-management.io/ocm/pkg/registration/hub/importer/providers/capi" "open-cluster-management.io/ocm/pkg/registration/hub/lease" "open-cluster-management.io/ocm/pkg/registration/hub/managedcluster" "open-cluster-management.io/ocm/pkg/registration/hub/managedclusterset" @@ -44,6 +48,7 @@ import ( type HubManagerOptions struct { ClusterAutoApprovalUsers []string GCResourceList []string + ImportOption *importeroptions.Options } // NewHubManagerOptions returns a HubManagerOptions @@ -51,6 +56,7 @@ func NewHubManagerOptions() *HubManagerOptions { return &HubManagerOptions{ GCResourceList: []string{"addon.open-cluster-management.io/v1alpha1/managedclusteraddons", "work.open-cluster-management.io/v1/manifestworks"}, + ImportOption: importeroptions.New(), } } @@ -62,6 +68,7 @@ func (m *HubManagerOptions) AddFlags(fs *pflag.FlagSet) { "A list GVR user can customize which are cleaned up after cluster is deleted. Format is group/version/resource, "+ "and the default are managedclusteraddon and manifestwork. The resources will be deleted in order."+ "The flag works only when ResourceCleanup feature gate is enable.") + m.ImportOption.AddFlags(fs) } // RunControllerManager starts the controllers on hub to manage spoke cluster registration. @@ -244,6 +251,23 @@ func (m *HubManagerOptions) RunControllerManagerWithInformers( ) } + var providers []cloudproviders.Interface + var clusterImporter factory.Controller + if features.HubMutableFeatureGate.Enabled(ocmfeature.ClusterImporter) { + providers = []cloudproviders.Interface{ + capi.NewCAPIProvider(controllerContext.KubeConfig, clusterInformers.Cluster().V1().ManagedClusters()), + } + clusterImporter = importer.NewImporter( + []importer.KlusterletConfigRenderer{ + importer.RenderBootstrapHubKubeConfig(kubeClient, m.ImportOption.APIServerURL), + }, + clusterClient, + clusterInformers.Cluster().V1().ManagedClusters(), + providers, + controllerContext.EventRecorder, + ) + } + gcController := gc.NewGCController( kubeInformers.Rbac().V1().ClusterRoles().Lister(), kubeInformers.Rbac().V1().ClusterRoleBindings().Lister(), @@ -284,6 +308,12 @@ func (m *HubManagerOptions) RunControllerManagerWithInformers( if features.HubMutableFeatureGate.Enabled(ocmfeature.ClusterProfile) { go clusterProfileController.Run(ctx, 1) } + if features.HubMutableFeatureGate.Enabled(ocmfeature.ClusterImporter) { + for _, provider := range providers { + go provider.Run(ctx) + } + go clusterImporter.Run(ctx, 1) + } go gcController.Run(ctx, 1) diff --git a/pkg/registration/hub/manager_test.go b/pkg/registration/hub/manager_test.go index 8b82e9bda..5309ac12b 100644 --- a/pkg/registration/hub/manager_test.go +++ b/pkg/registration/hub/manager_test.go @@ -71,6 +71,9 @@ var _ = ginkgo.BeforeSuite(func() { // enable resourceCleanup feature gate err = features.HubMutableFeatureGate.Set("ResourceCleanup=true") gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = features.HubMutableFeatureGate.Set("ClusterImporter=true") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) var _ = ginkgo.AfterSuite(func() { diff --git a/test/integration/registration/integration_suite_test.go b/test/integration/registration/integration_suite_test.go index db6b37aca..41e4c8165 100644 --- a/test/integration/registration/integration_suite_test.go +++ b/test/integration/registration/integration_suite_test.go @@ -75,6 +75,8 @@ var CRDPaths = []string{ "./vendor/open-cluster-management.io/api/cluster/v1beta2/0000_01_clusters.open-cluster-management.io_managedclustersetbindings.crd.yaml", // spoke "./vendor/open-cluster-management.io/api/cluster/v1alpha1/0000_02_clusters.open-cluster-management.io_clusterclaims.crd.yaml", + // external API deps + "./test/integration/testdeps/capi/cluster.x-k8s.io_clusters.yaml", } func runAgent(name string, opt *spoke.SpokeAgentOptions, commOption *commonoptions.AgentOptions, cfg *rest.Config) context.CancelFunc { @@ -196,12 +198,17 @@ var _ = ginkgo.BeforeSuite(func() { err = features.HubMutableFeatureGate.Set("ResourceCleanup=true") gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // enable clusterImporter feature gate + err = features.HubMutableFeatureGate.Set("ClusterImporter=true") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // start hub controller var ctx context.Context startHub = func() { ctx, stopHub = context.WithCancel(context.Background()) go func() { m := hub.NewHubManagerOptions() + m.ImportOption.APIServerURL = cfg.Host m.ClusterAutoApprovalUsers = []string{util.AutoApprovalBootstrapUser} err := m.RunControllerManager(ctx, &controllercmd.ControllerContext{ KubeConfig: cfg, diff --git a/test/integration/registration/managedcluster_importer_test.go b/test/integration/registration/managedcluster_importer_test.go new file mode 100644 index 000000000..4857e6429 --- /dev/null +++ b/test/integration/registration/managedcluster_importer_test.go @@ -0,0 +1,183 @@ +package registration_test + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/openshift/api" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/resource/resourceapply" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/rand" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/dynamic" + + operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned" + clusterv1 "open-cluster-management.io/api/cluster/v1" + operatorv1 "open-cluster-management.io/api/operator/v1" + + testingcommon "open-cluster-management.io/ocm/pkg/common/testing" + "open-cluster-management.io/ocm/pkg/operator/helpers/chart" + "open-cluster-management.io/ocm/pkg/registration/hub/importer" + "open-cluster-management.io/ocm/pkg/registration/hub/importer/providers/capi" + "open-cluster-management.io/ocm/test/integration/util" +) + +var ( + genericScheme = runtime.NewScheme() + genericCodecs = serializer.NewCodecFactory(genericScheme) + genericCodec = genericCodecs.UniversalDeserializer() +) + +func init() { + utilruntime.Must(api.InstallKube(genericScheme)) + utilruntime.Must(apiextensionsv1.AddToScheme(genericScheme)) + utilruntime.Must(operatorv1.Install(genericScheme)) +} + +var _ = ginkgo.Describe("Cluster Auto Importer", func() { + var managedClusterName string + var dynamicClient dynamic.Interface + var operatorClient operatorclient.Interface + ginkgo.BeforeEach(func() { + suffix := rand.String(5) + managedClusterName = fmt.Sprintf("managedcluster-%s", suffix) + + ginkgo.By("Create bootstrap token") + clusterManagerConfig := chart.NewDefaultClusterManagerChartConfig() + clusterManagerConfig.CreateBootstrapSA = true + clusterManagerConfig.CreateNamespace = true + manifests, err := chart.RenderClusterManagerChart(clusterManagerConfig, "open-cluster-management") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + recorder := events.NewInMemoryRecorder("importer-testing") + for _, manifest := range manifests { + requiredObj, _, err := genericCodec.Decode(manifest, nil, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + switch t := requiredObj.(type) { + case *corev1.Namespace: + _, _, err = resourceapply.ApplyNamespace(context.TODO(), kubeClient.CoreV1(), recorder, t) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + case *corev1.ServiceAccount: + _, _, err = resourceapply.ApplyServiceAccount(context.TODO(), kubeClient.CoreV1(), recorder, t) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + case *rbacv1.ClusterRole: + _, _, err = resourceapply.ApplyClusterRole(context.TODO(), kubeClient.RbacV1(), recorder, t) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + case *rbacv1.ClusterRoleBinding: + _, _, err = resourceapply.ApplyClusterRoleBinding(context.TODO(), kubeClient.RbacV1(), recorder, t) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + } + + dynamicClient, err = dynamic.NewForConfig(spokeCfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + operatorClient, err = operatorclient.NewForConfig(spokeCfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.Context("Cluster API importer", func() { + ginkgo.JustBeforeEach(func() { + cluster := &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedClusterName, + }, + Spec: clusterv1.ManagedClusterSpec{ + HubAcceptsClient: true, + }, + } + _, err := clusterClient.ClusterV1().ManagedClusters().Create(context.TODO(), cluster, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.JustAfterEach(func() { + err := clusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), managedClusterName, metav1.DeleteOptions{}) + if !errors.IsNotFound(err) { + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + }) + + ginkgo.It("Should import CAPI cluster", func() { + ginkgo.By("Create CAPI cluster") + capiCluster := testingcommon.NewUnstructured( + "cluster.x-k8s.io/v1beta1", "Cluster", managedClusterName, managedClusterName) + _, err := dynamicClient.Resource(capi.ClusterAPIGVR).Namespace(managedClusterName).Create( + context.TODO(), capiCluster, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + spokeCluster, err := util.GetManagedCluster(clusterClient, managedClusterName) + if err != nil { + return err + } + if !meta.IsStatusConditionFalse( + spokeCluster.Status.Conditions, importer.ManagedClusterConditionImported) { + return fmt.Errorf("cluster should have error when imported") + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + ginkgo.By("Create secret") + capiSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: managedClusterName + "-kubeconfig", + Namespace: managedClusterName, + }, + Data: map[string][]byte{ + "value": util.NewKubeConfig(spokeCfg), + }, + } + _, err = kubeClient.CoreV1().Secrets(managedClusterName).Create(context.TODO(), capiSecret, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // trigger the capi cluster resource to reconcile again + gomega.Eventually(func() error { + capiCluster, err := dynamicClient.Resource(capi.ClusterAPIGVR).Namespace(managedClusterName).Get( + context.TODO(), managedClusterName, metav1.GetOptions{}) + if err != nil { + return err + } + capiCluster.SetLabels(map[string]string{"reconcile": "trigger"}) + _, err = dynamicClient.Resource(capi.ClusterAPIGVR).Namespace(managedClusterName).Update( + context.TODO(), capiCluster, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + gomega.Eventually(func() error { + spokeCluster, err := util.GetManagedCluster(clusterClient, managedClusterName) + if err != nil { + return err + } + if !meta.IsStatusConditionTrue( + spokeCluster.Status.Conditions, importer.ManagedClusterConditionImported) { + return fmt.Errorf("cluster should have imported") + } + _, err = operatorClient.OperatorV1().Klusterlets().Get( + context.TODO(), "klusterlet", metav1.GetOptions{}) + if err != nil { + return err + } + return nil + }, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred()) + + err = dynamicClient.Resource(capi.ClusterAPIGVR).Namespace(managedClusterName).Delete( + context.TODO(), managedClusterName, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = kubeClient.CoreV1().Secrets(managedClusterName).Delete( + context.TODO(), managedClusterName+"-kubeconfig", metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + }) +}) diff --git a/test/integration/testdeps/capi/cluster.x-k8s.io_clusters.yaml b/test/integration/testdeps/capi/cluster.x-k8s.io_clusters.yaml new file mode 100644 index 000000000..aaa0a4114 --- /dev/null +++ b/test/integration/testdeps/capi/cluster.x-k8s.io_clusters.yaml @@ -0,0 +1,1939 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.1 + name: clusters.cluster.x-k8s.io +spec: + group: cluster.x-k8s.io + names: + categories: + - cluster-api + kind: Cluster + listKind: ClusterList + plural: clusters + shortNames: + - cl + singular: cluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Cluster status such as Pending/Provisioning/Provisioned/Deleting/Failed + jsonPath: .status.phase + name: Phase + type: string + deprecated: true + name: v1alpha3 + schema: + openAPIV3Schema: + description: Cluster is the Schema for the clusters API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterSpec defines the desired state of Cluster. + properties: + clusterNetwork: + description: Cluster network configuration. + properties: + apiServerPort: + description: |- + apiServerPort specifies the port the API Server should bind to. + Defaults to 6443. + format: int32 + type: integer + pods: + description: The network ranges from which Pod networks are allocated. + properties: + cidrBlocks: + items: + type: string + type: array + required: + - cidrBlocks + type: object + serviceDomain: + description: Domain name for services. + type: string + services: + description: The network ranges from which service VIPs are allocated. + properties: + cidrBlocks: + items: + type: string + type: array + required: + - cidrBlocks + type: object + type: object + controlPlaneEndpoint: + description: controlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + controlPlaneRef: + description: |- + controlPlaneRef is an optional reference to a provider-specific resource that holds + the details for provisioning the Control Plane for a Cluster. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + infrastructureRef: + description: |- + infrastructureRef is a reference to a provider-specific resource that holds the details + for provisioning infrastructure for a cluster in said provider. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + paused: + description: paused can be used to prevent controllers from processing + the Cluster and all its associated objects. + type: boolean + type: object + status: + description: ClusterStatus defines the observed state of Cluster. + properties: + conditions: + description: conditions defines current service state of the cluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. + type: string + severity: + description: |- + severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - status + - type + type: object + type: array + controlPlaneInitialized: + description: controlPlaneInitialized defines if the control plane + has been initialized. + type: boolean + controlPlaneReady: + description: controlPlaneReady defines if the control plane is ready. + type: boolean + failureDomains: + additionalProperties: + description: |- + FailureDomainSpec is the Schema for Cluster API failure domains. + It allows controllers to understand how many failure domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: controlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: failureDomains is a slice of failure domain objects synced + from the infrastructure provider. + type: object + failureMessage: + description: |- + failureMessage indicates that there is a fatal problem reconciling the + state, and will be set to a descriptive error message. + type: string + failureReason: + description: |- + failureReason indicates that there is a fatal problem reconciling the + state, and will be set to a token value suitable for + programmatic interpretation. + type: string + infrastructureReady: + description: infrastructureReady is the state of the infrastructure + provider. + type: boolean + observedGeneration: + description: observedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + description: |- + phase represents the current phase of cluster actuation. + E.g. Pending, Running, Terminating, Failed etc. + type: string + type: object + type: object + served: false + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Time duration since creation of Cluster + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Cluster status such as Pending/Provisioning/Provisioned/Deleting/Failed + jsonPath: .status.phase + name: Phase + type: string + deprecated: true + name: v1alpha4 + schema: + openAPIV3Schema: + description: |- + Cluster is the Schema for the clusters API. + + Deprecated: This type will be removed in one of the next releases. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterSpec defines the desired state of Cluster. + properties: + clusterNetwork: + description: Cluster network configuration. + properties: + apiServerPort: + description: |- + apiServerPort specifies the port the API Server should bind to. + Defaults to 6443. + format: int32 + type: integer + pods: + description: The network ranges from which Pod networks are allocated. + properties: + cidrBlocks: + items: + type: string + type: array + required: + - cidrBlocks + type: object + serviceDomain: + description: Domain name for services. + type: string + services: + description: The network ranges from which service VIPs are allocated. + properties: + cidrBlocks: + items: + type: string + type: array + required: + - cidrBlocks + type: object + type: object + controlPlaneEndpoint: + description: controlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + controlPlaneRef: + description: |- + controlPlaneRef is an optional reference to a provider-specific resource that holds + the details for provisioning the Control Plane for a Cluster. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + infrastructureRef: + description: |- + infrastructureRef is a reference to a provider-specific resource that holds the details + for provisioning infrastructure for a cluster in said provider. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + paused: + description: paused can be used to prevent controllers from processing + the Cluster and all its associated objects. + type: boolean + topology: + description: |- + This encapsulates the topology for the cluster. + NOTE: It is required to enable the ClusterTopology + feature gate flag to activate managed topologies support; + this feature is highly experimental, and parts of it might still be not implemented. + properties: + class: + description: The name of the ClusterClass object to create the + topology. + type: string + controlPlane: + description: controlPlane describes the cluster control plane. + properties: + metadata: + description: |- + metadata is the metadata applied to the machines of the ControlPlane. + At runtime this metadata is merged with the corresponding metadata from the ClusterClass. + + This field is supported if and only if the control plane provider template + referenced in the ClusterClass is Machine based. + properties: + annotations: + additionalProperties: + type: string + description: |- + annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + replicas: + description: |- + replicas is the number of control plane nodes. + If the value is nil, the ControlPlane object is created without the number of Replicas + and it's assumed that the control plane controller does not implement support for this field. + When specified against a control plane provider that lacks support for this field, this value will be ignored. + format: int32 + type: integer + type: object + rolloutAfter: + description: |- + rolloutAfter performs a rollout of the entire cluster one component at a time, + control plane first and then machine deployments. + format: date-time + type: string + version: + description: The Kubernetes version of the cluster. + type: string + workers: + description: |- + workers encapsulates the different constructs that form the worker nodes + for the cluster. + properties: + machineDeployments: + description: machineDeployments is a list of machine deployments + in the cluster. + items: + description: |- + MachineDeploymentTopology specifies the different parameters for a set of worker nodes in the topology. + This set of nodes is managed by a MachineDeployment object whose lifecycle is managed by the Cluster controller. + properties: + class: + description: |- + class is the name of the MachineDeploymentClass used to create the set of worker nodes. + This should match one of the deployment classes defined in the ClusterClass object + mentioned in the `Cluster.Spec.Class` field. + type: string + metadata: + description: |- + metadata is the metadata applied to the machines of the MachineDeployment. + At runtime this metadata is merged with the corresponding metadata from the ClusterClass. + properties: + annotations: + additionalProperties: + type: string + description: |- + annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + name: + description: |- + name is the unique identifier for this MachineDeploymentTopology. + The value is used with other unique identifiers to create a MachineDeployment's Name + (e.g. cluster's name, etc). In case the name is greater than the allowed maximum length, + the values are hashed together. + type: string + replicas: + description: |- + replicas is the number of worker nodes belonging to this set. + If the value is nil, the MachineDeployment is created without the number of Replicas (defaulting to zero) + and it's assumed that an external entity (like cluster autoscaler) is responsible for the management + of this value. + format: int32 + type: integer + required: + - class + - name + type: object + type: array + type: object + required: + - class + - version + type: object + type: object + status: + description: ClusterStatus defines the observed state of Cluster. + properties: + conditions: + description: conditions defines current service state of the cluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. + type: string + severity: + description: |- + severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - status + - type + type: object + type: array + controlPlaneReady: + description: controlPlaneReady defines if the control plane is ready. + type: boolean + failureDomains: + additionalProperties: + description: |- + FailureDomainSpec is the Schema for Cluster API failure domains. + It allows controllers to understand how many failure domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: controlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: failureDomains is a slice of failure domain objects synced + from the infrastructure provider. + type: object + failureMessage: + description: |- + failureMessage indicates that there is a fatal problem reconciling the + state, and will be set to a descriptive error message. + type: string + failureReason: + description: |- + failureReason indicates that there is a fatal problem reconciling the + state, and will be set to a token value suitable for + programmatic interpretation. + type: string + infrastructureReady: + description: infrastructureReady is the state of the infrastructure + provider. + type: boolean + observedGeneration: + description: observedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + description: |- + phase represents the current phase of cluster actuation. + E.g. Pending, Running, Terminating, Failed etc. + type: string + type: object + type: object + served: false + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: ClusterClass of this Cluster, empty if the Cluster is not using + a ClusterClass + jsonPath: .spec.topology.class + name: ClusterClass + type: string + - description: Cluster status such as Pending/Provisioning/Provisioned/Deleting/Failed + jsonPath: .status.phase + name: Phase + type: string + - description: Time duration since creation of Cluster + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Kubernetes version associated with this Cluster + jsonPath: .spec.topology.version + name: Version + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: Cluster is the Schema for the clusters API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterSpec defines the desired state of Cluster. + properties: + availabilityGates: + description: |- + availabilityGates specifies additional conditions to include when evaluating Cluster Available condition. + + NOTE: this field is considered only for computing v1beta2 conditions. + items: + description: ClusterAvailabilityGate contains the type of a Cluster + condition to be used as availability gate. + properties: + conditionType: + description: |- + conditionType refers to a positive polarity condition (status true means good) with matching type in the Cluster's condition list. + If the conditions doesn't exist, it will be treated as unknown. + Note: Both Cluster API conditions or conditions added by 3rd party controllers can be used as availability gates. + maxLength: 316 + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - conditionType + type: object + maxItems: 32 + type: array + x-kubernetes-list-map-keys: + - conditionType + x-kubernetes-list-type: map + clusterNetwork: + description: Cluster network configuration. + properties: + apiServerPort: + description: |- + apiServerPort specifies the port the API Server should bind to. + Defaults to 6443. + format: int32 + type: integer + pods: + description: The network ranges from which Pod networks are allocated. + properties: + cidrBlocks: + items: + type: string + type: array + required: + - cidrBlocks + type: object + serviceDomain: + description: Domain name for services. + type: string + services: + description: The network ranges from which service VIPs are allocated. + properties: + cidrBlocks: + items: + type: string + type: array + required: + - cidrBlocks + type: object + type: object + controlPlaneEndpoint: + description: controlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + controlPlaneRef: + description: |- + controlPlaneRef is an optional reference to a provider-specific resource that holds + the details for provisioning the Control Plane for a Cluster. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + infrastructureRef: + description: |- + infrastructureRef is a reference to a provider-specific resource that holds the details + for provisioning infrastructure for a cluster in said provider. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + paused: + description: paused can be used to prevent controllers from processing + the Cluster and all its associated objects. + type: boolean + topology: + description: |- + This encapsulates the topology for the cluster. + NOTE: It is required to enable the ClusterTopology + feature gate flag to activate managed topologies support; + this feature is highly experimental, and parts of it might still be not implemented. + properties: + class: + description: The name of the ClusterClass object to create the + topology. + type: string + controlPlane: + description: controlPlane describes the cluster control plane. + properties: + machineHealthCheck: + description: |- + machineHealthCheck allows to enable, disable and override + the MachineHealthCheck configuration in the ClusterClass for this control plane. + properties: + enable: + description: |- + enable controls if a MachineHealthCheck should be created for the target machines. + + If false: No MachineHealthCheck will be created. + + If not set(default): A MachineHealthCheck will be created if it is defined here or + in the associated ClusterClass. If no MachineHealthCheck is defined then none will be created. + + If true: A MachineHealthCheck is guaranteed to be created. Cluster validation will + block if `enable` is true and no MachineHealthCheck definition is available. + type: boolean + maxUnhealthy: + anyOf: + - type: integer + - type: string + description: |- + Any further remediation is only allowed if at most "MaxUnhealthy" machines selected by + "selector" are not healthy. + x-kubernetes-int-or-string: true + nodeStartupTimeout: + description: |- + nodeStartupTimeout allows to set the maximum time for MachineHealthCheck + to consider a Machine unhealthy if a corresponding Node isn't associated + through a `Spec.ProviderID` field. + + The duration set in this field is compared to the greatest of: + - Cluster's infrastructure ready condition timestamp (if and when available) + - Control Plane's initialized condition timestamp (if and when available) + - Machine's infrastructure ready condition timestamp (if and when available) + - Machine's metadata creation timestamp + + Defaults to 10 minutes. + If you wish to disable this feature, set the value explicitly to 0. + type: string + remediationTemplate: + description: |- + remediationTemplate is a reference to a remediation template + provided by an infrastructure provider. + + This field is completely optional, when filled, the MachineHealthCheck controller + creates a new object from the template referenced and hands off remediation of the machine to + a controller that lives outside of Cluster API. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + unhealthyConditions: + description: |- + unhealthyConditions contains a list of the conditions that determine + whether a node is considered unhealthy. The conditions are combined in a + logical OR, i.e. if any of the conditions is met, the node is unhealthy. + items: + description: |- + UnhealthyCondition represents a Node condition type and value with a timeout + specified as a duration. When the named condition has been in the given + status for at least the timeout value, a node is considered unhealthy. + properties: + status: + minLength: 1 + type: string + timeout: + type: string + type: + minLength: 1 + type: string + required: + - status + - timeout + - type + type: object + type: array + unhealthyRange: + description: |- + Any further remediation is only allowed if the number of machines selected by "selector" as not healthy + is within the range of "UnhealthyRange". Takes precedence over MaxUnhealthy. + Eg. "[3-5]" - This means that remediation will be allowed only when: + (a) there are at least 3 unhealthy machines (and) + (b) there are at most 5 unhealthy machines + pattern: ^\[[0-9]+-[0-9]+\]$ + type: string + type: object + metadata: + description: |- + metadata is the metadata applied to the ControlPlane and the Machines of the ControlPlane + if the ControlPlaneTemplate referenced by the ClusterClass is machine based. If not, it + is applied only to the ControlPlane. + At runtime this metadata is merged with the corresponding metadata from the ClusterClass. + properties: + annotations: + additionalProperties: + type: string + description: |- + annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + nodeDeletionTimeout: + description: |- + nodeDeletionTimeout defines how long the controller will attempt to delete the Node that the Machine + hosts after the Machine is marked for deletion. A duration of 0 will retry deletion indefinitely. + Defaults to 10 seconds. + type: string + nodeDrainTimeout: + description: |- + nodeDrainTimeout is the total amount of time that the controller will spend on draining a node. + The default value is 0, meaning that the node can be drained without any time limitations. + NOTE: NodeDrainTimeout is different from `kubectl drain --timeout` + type: string + nodeVolumeDetachTimeout: + description: |- + nodeVolumeDetachTimeout is the total amount of time that the controller will spend on waiting for all volumes + to be detached. The default value is 0, meaning that the volumes can be detached without any time limitations. + type: string + replicas: + description: |- + replicas is the number of control plane nodes. + If the value is nil, the ControlPlane object is created without the number of Replicas + and it's assumed that the control plane controller does not implement support for this field. + When specified against a control plane provider that lacks support for this field, this value will be ignored. + format: int32 + type: integer + variables: + description: variables can be used to customize the ControlPlane + through patches. + properties: + overrides: + description: overrides can be used to override Cluster + level variables. + items: + description: |- + ClusterVariable can be used to customize the Cluster through patches. Each ClusterVariable is associated with a + Variable definition in the ClusterClass `status` variables. + properties: + definitionFrom: + description: |- + definitionFrom specifies where the definition of this Variable is from. + + Deprecated: This field is deprecated, must not be set anymore and is going to be removed in the next apiVersion. + type: string + name: + description: name of the variable. + type: string + value: + description: |- + value of the variable. + Note: the value will be validated against the schema of the corresponding ClusterClassVariable + from the ClusterClass. + Note: We have to use apiextensionsv1.JSON instead of a custom JSON type, because controller-tools has a + hard-coded schema for apiextensionsv1.JSON which cannot be produced by another type via controller-tools, + i.e. it is not possible to have no type field. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/d0e03a142d0ecdd5491593e941ee1d6b5d91dba6/pkg/crd/known_types.go#L106-L111 + x-kubernetes-preserve-unknown-fields: true + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: object + rolloutAfter: + description: |- + rolloutAfter performs a rollout of the entire cluster one component at a time, + control plane first and then machine deployments. + + Deprecated: This field has no function and is going to be removed in the next apiVersion. + format: date-time + type: string + variables: + description: |- + variables can be used to customize the Cluster through + patches. They must comply to the corresponding + VariableClasses defined in the ClusterClass. + items: + description: |- + ClusterVariable can be used to customize the Cluster through patches. Each ClusterVariable is associated with a + Variable definition in the ClusterClass `status` variables. + properties: + definitionFrom: + description: |- + definitionFrom specifies where the definition of this Variable is from. + + Deprecated: This field is deprecated, must not be set anymore and is going to be removed in the next apiVersion. + type: string + name: + description: name of the variable. + type: string + value: + description: |- + value of the variable. + Note: the value will be validated against the schema of the corresponding ClusterClassVariable + from the ClusterClass. + Note: We have to use apiextensionsv1.JSON instead of a custom JSON type, because controller-tools has a + hard-coded schema for apiextensionsv1.JSON which cannot be produced by another type via controller-tools, + i.e. it is not possible to have no type field. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/d0e03a142d0ecdd5491593e941ee1d6b5d91dba6/pkg/crd/known_types.go#L106-L111 + x-kubernetes-preserve-unknown-fields: true + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + version: + description: The Kubernetes version of the cluster. + type: string + workers: + description: |- + workers encapsulates the different constructs that form the worker nodes + for the cluster. + properties: + machineDeployments: + description: machineDeployments is a list of machine deployments + in the cluster. + items: + description: |- + MachineDeploymentTopology specifies the different parameters for a set of worker nodes in the topology. + This set of nodes is managed by a MachineDeployment object whose lifecycle is managed by the Cluster controller. + properties: + class: + description: |- + class is the name of the MachineDeploymentClass used to create the set of worker nodes. + This should match one of the deployment classes defined in the ClusterClass object + mentioned in the `Cluster.Spec.Class` field. + type: string + failureDomain: + description: |- + failureDomain is the failure domain the machines will be created in. + Must match a key in the FailureDomains map stored on the cluster object. + type: string + machineHealthCheck: + description: |- + machineHealthCheck allows to enable, disable and override + the MachineHealthCheck configuration in the ClusterClass for this MachineDeployment. + properties: + enable: + description: |- + enable controls if a MachineHealthCheck should be created for the target machines. + + If false: No MachineHealthCheck will be created. + + If not set(default): A MachineHealthCheck will be created if it is defined here or + in the associated ClusterClass. If no MachineHealthCheck is defined then none will be created. + + If true: A MachineHealthCheck is guaranteed to be created. Cluster validation will + block if `enable` is true and no MachineHealthCheck definition is available. + type: boolean + maxUnhealthy: + anyOf: + - type: integer + - type: string + description: |- + Any further remediation is only allowed if at most "MaxUnhealthy" machines selected by + "selector" are not healthy. + x-kubernetes-int-or-string: true + nodeStartupTimeout: + description: |- + nodeStartupTimeout allows to set the maximum time for MachineHealthCheck + to consider a Machine unhealthy if a corresponding Node isn't associated + through a `Spec.ProviderID` field. + + The duration set in this field is compared to the greatest of: + - Cluster's infrastructure ready condition timestamp (if and when available) + - Control Plane's initialized condition timestamp (if and when available) + - Machine's infrastructure ready condition timestamp (if and when available) + - Machine's metadata creation timestamp + + Defaults to 10 minutes. + If you wish to disable this feature, set the value explicitly to 0. + type: string + remediationTemplate: + description: |- + remediationTemplate is a reference to a remediation template + provided by an infrastructure provider. + + This field is completely optional, when filled, the MachineHealthCheck controller + creates a new object from the template referenced and hands off remediation of the machine to + a controller that lives outside of Cluster API. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + unhealthyConditions: + description: |- + unhealthyConditions contains a list of the conditions that determine + whether a node is considered unhealthy. The conditions are combined in a + logical OR, i.e. if any of the conditions is met, the node is unhealthy. + items: + description: |- + UnhealthyCondition represents a Node condition type and value with a timeout + specified as a duration. When the named condition has been in the given + status for at least the timeout value, a node is considered unhealthy. + properties: + status: + minLength: 1 + type: string + timeout: + type: string + type: + minLength: 1 + type: string + required: + - status + - timeout + - type + type: object + type: array + unhealthyRange: + description: |- + Any further remediation is only allowed if the number of machines selected by "selector" as not healthy + is within the range of "UnhealthyRange". Takes precedence over MaxUnhealthy. + Eg. "[3-5]" - This means that remediation will be allowed only when: + (a) there are at least 3 unhealthy machines (and) + (b) there are at most 5 unhealthy machines + pattern: ^\[[0-9]+-[0-9]+\]$ + type: string + type: object + metadata: + description: |- + metadata is the metadata applied to the MachineDeployment and the machines of the MachineDeployment. + At runtime this metadata is merged with the corresponding metadata from the ClusterClass. + properties: + annotations: + additionalProperties: + type: string + description: |- + annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + minReadySeconds: + description: |- + Minimum number of seconds for which a newly created machine should + be ready. + Defaults to 0 (machine will be considered available as soon as it + is ready) + format: int32 + type: integer + name: + description: |- + name is the unique identifier for this MachineDeploymentTopology. + The value is used with other unique identifiers to create a MachineDeployment's Name + (e.g. cluster's name, etc). In case the name is greater than the allowed maximum length, + the values are hashed together. + type: string + nodeDeletionTimeout: + description: |- + nodeDeletionTimeout defines how long the controller will attempt to delete the Node that the Machine + hosts after the Machine is marked for deletion. A duration of 0 will retry deletion indefinitely. + Defaults to 10 seconds. + type: string + nodeDrainTimeout: + description: |- + nodeDrainTimeout is the total amount of time that the controller will spend on draining a node. + The default value is 0, meaning that the node can be drained without any time limitations. + NOTE: NodeDrainTimeout is different from `kubectl drain --timeout` + type: string + nodeVolumeDetachTimeout: + description: |- + nodeVolumeDetachTimeout is the total amount of time that the controller will spend on waiting for all volumes + to be detached. The default value is 0, meaning that the volumes can be detached without any time limitations. + type: string + replicas: + description: |- + replicas is the number of worker nodes belonging to this set. + If the value is nil, the MachineDeployment is created without the number of Replicas (defaulting to 1) + and it's assumed that an external entity (like cluster autoscaler) is responsible for the management + of this value. + format: int32 + type: integer + strategy: + description: |- + The deployment strategy to use to replace existing machines with + new ones. + properties: + remediation: + description: |- + remediation controls the strategy of remediating unhealthy machines + and how remediating operations should occur during the lifecycle of the dependant MachineSets. + properties: + maxInFlight: + anyOf: + - type: integer + - type: string + description: |- + maxInFlight determines how many in flight remediations should happen at the same time. + + Remediation only happens on the MachineSet with the most current revision, while + older MachineSets (usually present during rollout operations) aren't allowed to remediate. + + Note: In general (independent of remediations), unhealthy machines are always + prioritized during scale down operations over healthy ones. + + MaxInFlight can be set to a fixed number or a percentage. + Example: when this is set to 20%, the MachineSet controller deletes at most 20% of + the desired replicas. + + If not set, remediation is limited to all machines (bounded by replicas) + under the active MachineSet's management. + x-kubernetes-int-or-string: true + type: object + rollingUpdate: + description: |- + Rolling update config params. Present only if + MachineDeploymentStrategyType = RollingUpdate. + properties: + deletePolicy: + description: |- + deletePolicy defines the policy used by the MachineDeployment to identify nodes to delete when downscaling. + Valid values are "Random, "Newest", "Oldest" + When no value is supplied, the default DeletePolicy of MachineSet is used + enum: + - Random + - Newest + - Oldest + type: string + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of machines that can be scheduled above the + desired number of machines. + Value can be an absolute number (ex: 5) or a percentage of + desired machines (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 1. + Example: when this is set to 30%, the new MachineSet can be scaled + up immediately when the rolling update starts, such that the total + number of old and new machines do not exceed 130% of desired + machines. Once old machines have been killed, new MachineSet can + be scaled up further, ensuring that total number of machines running + at any time during the update is at most 130% of desired machines. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of machines that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired + machines (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 0. + Example: when this is set to 30%, the old MachineSet can be scaled + down to 70% of desired machines immediately when the rolling update + starts. Once new machines are ready, old MachineSet can be scaled + down further, followed by scaling up the new MachineSet, ensuring + that the total number of machines available at all times + during the update is at least 70% of desired machines. + x-kubernetes-int-or-string: true + type: object + type: + description: |- + type of deployment. Allowed values are RollingUpdate and OnDelete. + The default is RollingUpdate. + enum: + - RollingUpdate + - OnDelete + type: string + type: object + variables: + description: variables can be used to customize the + MachineDeployment through patches. + properties: + overrides: + description: overrides can be used to override Cluster + level variables. + items: + description: |- + ClusterVariable can be used to customize the Cluster through patches. Each ClusterVariable is associated with a + Variable definition in the ClusterClass `status` variables. + properties: + definitionFrom: + description: |- + definitionFrom specifies where the definition of this Variable is from. + + Deprecated: This field is deprecated, must not be set anymore and is going to be removed in the next apiVersion. + type: string + name: + description: name of the variable. + type: string + value: + description: |- + value of the variable. + Note: the value will be validated against the schema of the corresponding ClusterClassVariable + from the ClusterClass. + Note: We have to use apiextensionsv1.JSON instead of a custom JSON type, because controller-tools has a + hard-coded schema for apiextensionsv1.JSON which cannot be produced by another type via controller-tools, + i.e. it is not possible to have no type field. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/d0e03a142d0ecdd5491593e941ee1d6b5d91dba6/pkg/crd/known_types.go#L106-L111 + x-kubernetes-preserve-unknown-fields: true + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - class + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + machinePools: + description: machinePools is a list of machine pools in the + cluster. + items: + description: |- + MachinePoolTopology specifies the different parameters for a pool of worker nodes in the topology. + This pool of nodes is managed by a MachinePool object whose lifecycle is managed by the Cluster controller. + properties: + class: + description: |- + class is the name of the MachinePoolClass used to create the pool of worker nodes. + This should match one of the deployment classes defined in the ClusterClass object + mentioned in the `Cluster.Spec.Class` field. + type: string + failureDomains: + description: |- + failureDomains is the list of failure domains the machine pool will be created in. + Must match a key in the FailureDomains map stored on the cluster object. + items: + type: string + type: array + metadata: + description: |- + metadata is the metadata applied to the MachinePool. + At runtime this metadata is merged with the corresponding metadata from the ClusterClass. + properties: + annotations: + additionalProperties: + type: string + description: |- + annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + minReadySeconds: + description: |- + Minimum number of seconds for which a newly created machine pool should + be ready. + Defaults to 0 (machine will be considered available as soon as it + is ready) + format: int32 + type: integer + name: + description: |- + name is the unique identifier for this MachinePoolTopology. + The value is used with other unique identifiers to create a MachinePool's Name + (e.g. cluster's name, etc). In case the name is greater than the allowed maximum length, + the values are hashed together. + type: string + nodeDeletionTimeout: + description: |- + nodeDeletionTimeout defines how long the controller will attempt to delete the Node that the MachinePool + hosts after the MachinePool is marked for deletion. A duration of 0 will retry deletion indefinitely. + Defaults to 10 seconds. + type: string + nodeDrainTimeout: + description: |- + nodeDrainTimeout is the total amount of time that the controller will spend on draining a node. + The default value is 0, meaning that the node can be drained without any time limitations. + NOTE: NodeDrainTimeout is different from `kubectl drain --timeout` + type: string + nodeVolumeDetachTimeout: + description: |- + nodeVolumeDetachTimeout is the total amount of time that the controller will spend on waiting for all volumes + to be detached. The default value is 0, meaning that the volumes can be detached without any time limitations. + type: string + replicas: + description: |- + replicas is the number of nodes belonging to this pool. + If the value is nil, the MachinePool is created without the number of Replicas (defaulting to 1) + and it's assumed that an external entity (like cluster autoscaler) is responsible for the management + of this value. + format: int32 + type: integer + variables: + description: variables can be used to customize the + MachinePool through patches. + properties: + overrides: + description: overrides can be used to override Cluster + level variables. + items: + description: |- + ClusterVariable can be used to customize the Cluster through patches. Each ClusterVariable is associated with a + Variable definition in the ClusterClass `status` variables. + properties: + definitionFrom: + description: |- + definitionFrom specifies where the definition of this Variable is from. + + Deprecated: This field is deprecated, must not be set anymore and is going to be removed in the next apiVersion. + type: string + name: + description: name of the variable. + type: string + value: + description: |- + value of the variable. + Note: the value will be validated against the schema of the corresponding ClusterClassVariable + from the ClusterClass. + Note: We have to use apiextensionsv1.JSON instead of a custom JSON type, because controller-tools has a + hard-coded schema for apiextensionsv1.JSON which cannot be produced by another type via controller-tools, + i.e. it is not possible to have no type field. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/d0e03a142d0ecdd5491593e941ee1d6b5d91dba6/pkg/crd/known_types.go#L106-L111 + x-kubernetes-preserve-unknown-fields: true + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - class + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - class + - version + type: object + type: object + status: + description: ClusterStatus defines the observed state of Cluster. + properties: + conditions: + description: conditions defines current service state of the cluster. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may be empty. + type: string + severity: + description: |- + severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + controlPlaneReady: + description: |- + controlPlaneReady denotes if the control plane became ready during initial provisioning + to receive requests. + NOTE: this field is part of the Cluster API contract and it is used to orchestrate provisioning. + The value of this field is never updated after provisioning is completed. Please use conditions + to check the operational state of the control plane. + type: boolean + failureDomains: + additionalProperties: + description: |- + FailureDomainSpec is the Schema for Cluster API failure domains. + It allows controllers to understand how many failure domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: controlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: failureDomains is a slice of failure domain objects synced + from the infrastructure provider. + type: object + failureMessage: + description: |- + failureMessage indicates that there is a fatal problem reconciling the + state, and will be set to a descriptive error message. + + Deprecated: This field is deprecated and is going to be removed in the next apiVersion. Please see https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20240916-improve-status-in-CAPI-resources.md for more details. + type: string + failureReason: + description: |- + failureReason indicates that there is a fatal problem reconciling the + state, and will be set to a token value suitable for + programmatic interpretation. + + Deprecated: This field is deprecated and is going to be removed in the next apiVersion. Please see https://github.com/kubernetes-sigs/cluster-api/blob/main/docs/proposals/20240916-improve-status-in-CAPI-resources.md for more details. + type: string + infrastructureReady: + description: infrastructureReady is the state of the infrastructure + provider. + type: boolean + observedGeneration: + description: observedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + phase: + description: |- + phase represents the current phase of cluster actuation. + E.g. Pending, Running, Terminating, Failed etc. + type: string + v1beta2: + description: v1beta2 groups all the fields that will be added or modified + in Cluster's status with the V1Beta2 version. + properties: + conditions: + description: |- + conditions represents the observations of a Cluster's current state. + Known condition types are Available, InfrastructureReady, ControlPlaneInitialized, ControlPlaneAvailable, WorkersAvailable, MachinesReady + MachinesUpToDate, RemoteConnectionProbe, ScalingUp, ScalingDown, Remediating, Deleting, Paused. + Additionally, a TopologyReconciled condition will be added in case the Cluster is referencing a ClusterClass / defining a managed Topology. + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 32 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controlPlane: + description: controlPlane groups all the observations about Cluster's + ControlPlane current state. + properties: + availableReplicas: + description: availableReplicas is the total number of available + control plane machines in this cluster. A machine is considered + available when Machine's Available condition is true. + format: int32 + type: integer + desiredReplicas: + description: desiredReplicas is the total number of desired + control plane machines in this cluster. + format: int32 + type: integer + readyReplicas: + description: readyReplicas is the total number of ready control + plane machines in this cluster. A machine is considered + ready when Machine's Ready condition is true. + format: int32 + type: integer + replicas: + description: |- + replicas is the total number of control plane machines in this cluster. + NOTE: replicas also includes machines still being provisioned or being deleted. + format: int32 + type: integer + upToDateReplicas: + description: upToDateReplicas is the number of up-to-date + control plane machines in this cluster. A machine is considered + up-to-date when Machine's UpToDate condition is true. + format: int32 + type: integer + type: object + workers: + description: workers groups all the observations about Cluster's + Workers current state. + properties: + availableReplicas: + description: availableReplicas is the total number of available + worker machines in this cluster. A machine is considered + available when Machine's Available condition is true. + format: int32 + type: integer + desiredReplicas: + description: desiredReplicas is the total number of desired + worker machines in this cluster. + format: int32 + type: integer + readyReplicas: + description: readyReplicas is the total number of ready worker + machines in this cluster. A machine is considered ready + when Machine's Ready condition is true. + format: int32 + type: integer + replicas: + description: |- + replicas is the total number of worker machines in this cluster. + NOTE: replicas also includes machines still being provisioned or being deleted. + format: int32 + type: integer + upToDateReplicas: + description: upToDateReplicas is the number of up-to-date + worker machines in this cluster. A machine is considered + up-to-date when Machine's UpToDate condition is true. + format: int32 + type: integer + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/vendor/modules.txt b/vendor/modules.txt index 32c35f3f9..653224b54 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1584,7 +1584,7 @@ open-cluster-management.io/addon-framework/pkg/agent open-cluster-management.io/addon-framework/pkg/assets open-cluster-management.io/addon-framework/pkg/index open-cluster-management.io/addon-framework/pkg/utils -# open-cluster-management.io/api v0.15.1-0.20241209025232-b62746ae96d4 +# open-cluster-management.io/api v0.15.1-0.20241210025410-0ba6809d0ae2 ## explicit; go 1.22.0 open-cluster-management.io/api/addon/v1alpha1 open-cluster-management.io/api/client/addon/clientset/versioned diff --git a/vendor/open-cluster-management.io/api/feature/feature.go b/vendor/open-cluster-management.io/api/feature/feature.go index 1e658be5a..81a8384df 100644 --- a/vendor/open-cluster-management.io/api/feature/feature.go +++ b/vendor/open-cluster-management.io/api/feature/feature.go @@ -80,6 +80,9 @@ const ( // ClusterProfile will start new controller in the Hub that can be used to sync ManagedCluster to ClusterProfile. ClusterProfile featuregate.Feature = "ClusterProfile" + + // ClusterImporter will enable the auto import of managed cluster for certain cluster providers, e.g. cluster-api. + ClusterImporter featuregate.Feature = "ClusterImporter" ) // DefaultSpokeRegistrationFeatureGates consists of all known ocm-registration @@ -87,7 +90,7 @@ const ( // add it here. var DefaultSpokeRegistrationFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ ClusterClaim: {Default: true, PreRelease: featuregate.Beta}, - AddonManagement: {Default: true, PreRelease: featuregate.Alpha}, + AddonManagement: {Default: true, PreRelease: featuregate.Beta}, V1beta1CSRAPICompatibility: {Default: false, PreRelease: featuregate.Alpha}, MultipleHubs: {Default: false, PreRelease: featuregate.Alpha}, } @@ -101,10 +104,11 @@ var DefaultHubRegistrationFeatureGates = map[featuregate.Feature]featuregate.Fea ManagedClusterAutoApproval: {Default: false, PreRelease: featuregate.Alpha}, ResourceCleanup: {Default: false, PreRelease: featuregate.Alpha}, ClusterProfile: {Default: false, PreRelease: featuregate.Alpha}, + ClusterImporter: {Default: false, PreRelease: featuregate.Alpha}, } var DefaultHubAddonManagerFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - AddonManagement: {Default: true, PreRelease: featuregate.Alpha}, + AddonManagement: {Default: true, PreRelease: featuregate.Beta}, } // DefaultHubWorkFeatureGates consists of all known acm work wehbook feature keys.