Merge pull request #19 from skeeey/adding-integration-test

add integration test cases

test/integration/certificate_rotation_test.go

@@ -0,0 +1,84 @@
+package integration_test
+
+import (
+	"context"
+	"path"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	"github.com/open-cluster-management/registration/pkg/spoke"
+	"github.com/open-cluster-management/registration/test/integration/util"
+
+	"github.com/openshift/library-go/pkg/controller/controllercmd"
+)
+
+var _ = ginkgo.Describe("Certificate Rotation", func() {
+	ginkgo.It("Certificate should be automatically rotated when it is about to expire", func() {
+		var err error
+
+		spokeClusterName := "rotationtest-spokecluster"
+		hubKubeconfigSecret := "rotationtest-hub-kubeconfig-secret"
+		hubKubeconfigDir := path.Join(util.TestDir, "rotationtest", "hub-kubeconfig")
+
+		// run registration agent
+		go func() {
+			agentOptions := spoke.SpokeAgentOptions{
+				ClusterName:         spokeClusterName,
+				BootstrapKubeconfig: bootstrapKubeConfigFile,
+				HubKubeconfigSecret: hubKubeconfigSecret,
+				HubKubeconfigDir:    hubKubeconfigDir,
+			}
+			err := agentOptions.RunSpokeAgent(context.Background(), &controllercmd.ControllerContext{
+				KubeConfig:    spokeCfg,
+				EventRecorder: util.NewIntegrationTestEventRecorder("rotationtest"),
+			})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}()
+
+		// after bootstrap the spokecluster and csr should be created
+		gomega.Eventually(func() bool {
+			if _, err := util.GetSpokeCluster(clusterClient, spokeClusterName); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		gomega.Eventually(func() bool {
+			if _, err := util.FindUnapprovedSpokeCSR(kubeClient, spokeClusterName); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// simulate hub cluster admin approve the csr with a short time certificate
+		err = util.ApproveSpokeClusterCSR(kubeClient, spokeClusterName, time.Second*20)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// simulate hub cluster admin accept the spokecluster
+		err = util.AcceptSpokeCluster(clusterClient, spokeClusterName)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// the hub kubeconfig secret should be filled after the csr is approved
+		gomega.Eventually(func() bool {
+			if _, err := util.GetFilledHubKubeConfigSecret(kubeClient, testNamespace, hubKubeconfigSecret); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// simulate k8s to mount the hub kubeconfig secret
+		err = util.MountHubKubeConfigs(kubeClient, hubKubeconfigDir, testNamespace, hubKubeconfigSecret)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// the agent should rotate the certificate because the certificate with a short valid time
+		// the hub controller should auto approve it
+		gomega.Eventually(func() bool {
+			if _, err := util.FindAutoApprovedSpokeCSR(kubeClient, spokeClusterName); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+	})
+})

test/integration/doc.go

@@ -0,0 +1,6 @@
+// Package integration provides integration tests for open-cluster-management registration, the test cases include
+// - spoke cluster joining process
+// - spoke registration rotate its certificate after its certificate is expired
+// - spoke registration agent recovery from invalid bootstrap kubeconfig
+// - spoke registration agent recovery from invalid hub kubeconfig
+package integration

test/integration/integration_suite_test.go

@@ -1,62 +1,147 @@
 package integration_test
 
 import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
 	"path/filepath"
 	"testing"
+	"time"
 
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	"github.com/openshift/library-go/pkg/controller/controllercmd"
+
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/transport"
 
+	clusterclientset "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
 	clusterv1 "github.com/open-cluster-management/api/cluster/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
+	"github.com/open-cluster-management/registration/pkg/hub"
+	"github.com/open-cluster-management/registration/pkg/spoke/hubclientcert"
+	"github.com/open-cluster-management/registration/pkg/spoke/spokecluster"
+	"github.com/open-cluster-management/registration/test/integration/util"
+
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
-var cfg *rest.Config
+const (
+	eventuallyTimeout  = 30 // seconds
+	eventuallyInterval = 1  // seconds
+)
+
+var spokeCfg *rest.Config
+var bootstrapKubeConfigFile string
+
 var testEnv *envtest.Environment
-var k8sClient client.Client
+var securePort int
+
+var kubeClient kubernetes.Interface
+var clusterClient clusterclientset.Interface
+
+var testNamespace string
 
 func TestIntegration(t *testing.T) {
-	RegisterFailHandler(Fail)
-	// TODO test cases
-	// - spoke registration agent creates CSR, hub authorizes, spoke agent creates hub kubeconfig and connects back to hub for successful join
-	// - spoke registration agent recovery from invalid bootstrap kubeconfig
-	// - spoke registration agent recovery from invalid hub kubeconfig
-	// - spoke registration rotate its certificate after its certificate is expired
-	RunSpecsWithDefaultAndCustomReporters(t, "Integration Suite", []Reporter{printer.NewlineReporter{}})
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Integration Suite", []ginkgo.Reporter{printer.NewlineReporter{}})
 }
 
-var _ = BeforeSuite(func(done Done) {
-	logf.SetLogger(zap.LoggerTo(GinkgoWriter, true))
+var _ = ginkgo.BeforeSuite(func(done ginkgo.Done) {
+	logf.SetLogger(zap.LoggerTo(ginkgo.GinkgoWriter, true))
 
-	By("bootstrapping test environment")
+	ginkgo.By("bootstrapping test environment")
+
+	var err error
+
+	// crank up the sync speed
+	transport.CertCallbackRefreshDuration = 5 * time.Second
+	hubclientcert.ControllerSyncInterval = 5 * time.Second
+	spokecluster.CreatingControllerSyncInterval = 1 * time.Second
 
 	// install cluster CRD and start a local kube-apiserver
+
+	err = util.GenerateSelfSignedCertKey()
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+	apiServerFlags := append([]string{}, envtest.DefaultKubeAPIServerFlags...)
+	apiServerFlags = append(apiServerFlags,
+		fmt.Sprintf("--client-ca-file=%s", util.CAFile),
+		fmt.Sprintf("--tls-cert-file=%s", util.ServerCertFile),
+		fmt.Sprintf("--tls-private-key-file=%s", util.ServerKeyFile),
+	)
+
 	testEnv = &envtest.Environment{
+		ErrorIfCRDPathMissing: true,
 		CRDDirectoryPaths: []string{
-			filepath.Join("..", "..", "vendor", "github.com", "open-cluster-management", "api", "cluster", "v1"),
+			filepath.Join(".", "vendor", "github.com", "open-cluster-management", "api", "cluster", "v1"),
 		},
+		KubeAPIServerFlags: apiServerFlags,
 	}
+
 	cfg, err := testEnv.Start()
-	Expect(err).ToNot(HaveOccurred())
-	Expect(cfg).ToNot(BeNil())
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	gomega.Expect(cfg).ToNot(gomega.BeNil())
 
 	err = clusterv1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	// prepare configs
+	securePort = testEnv.ControlPlane.APIServer.SecurePort
+	gomega.Expect(securePort).ToNot(gomega.BeZero())
+
+	spokeCfg, err = util.CreateSpokeKubeConfig(cfg, securePort)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	gomega.Expect(spokeCfg).ToNot(gomega.BeNil())
+
+	bootstrapKubeConfigFile = path.Join(util.TestDir, "bootstrap", "kubeconfig")
+	err = util.CreateBootstrapKubeConfig(bootstrapKubeConfigFile, securePort)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	// prepare clients
+	kubeClient, err = kubernetes.NewForConfig(cfg)
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	gomega.Expect(kubeClient).ToNot(gomega.BeNil())
+
+	clusterClient, err = clusterclientset.NewForConfig(cfg)
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	gomega.Expect(clusterClient).ToNot(gomega.BeNil())
+
+	// prepare test namespace
+	nsBytes, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+	if err != nil {
+		testNamespace = "open-cluster-management"
+	} else {
+		testNamespace = string(nsBytes)
+	}
+	err = util.PrepareSpokeAgentNamespace(kubeClient, testNamespace)
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+	// start hub controller
+	go func() {
+		err := hub.RunControllerManager(context.Background(), &controllercmd.ControllerContext{
+			KubeConfig:    cfg,
+			EventRecorder: util.NewIntegrationTestEventRecorder("hub"),
+		})
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	}()
 
-	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
-	Expect(err).ToNot(HaveOccurred())
-	Expect(k8sClient).ToNot(BeNil())
 	close(done)
 }, 60)
 
-var _ = AfterSuite(func() {
-	By("tearing down the test environment")
+var _ = ginkgo.AfterSuite(func() {
+	ginkgo.By("tearing down the test environment")
+
 	err := testEnv.Stop()
-	Expect(err).ToNot(HaveOccurred())
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
+	err = os.RemoveAll(util.TestDir)
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 })

test/integration/spokeagent_recovery_test.go

@@ -0,0 +1,214 @@
+package integration_test
+
+import (
+	"context"
+	"path"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	clusterv1 "github.com/open-cluster-management/api/cluster/v1"
+	"github.com/open-cluster-management/registration/pkg/helpers"
+	"github.com/open-cluster-management/registration/pkg/spoke"
+	"github.com/open-cluster-management/registration/test/integration/util"
+
+	"github.com/openshift/library-go/pkg/controller/controllercmd"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+)
+
+var _ = ginkgo.Describe("Agent Recovery", func() {
+	ginkgo.It("agent recovery from invalid bootstrap kubeconfig", func() {
+		var err error
+
+		spokeClusterName := "bootstrap-recoverytest-spokecluster"
+
+		hubKubeconfigSecret := "bootstrap-recoverytest-hub-kubeconfig-secret"
+		hubKubeconfigDir := path.Join(util.TestDir, "bootstrap-recoverytest", "hub-kubeconfig")
+
+		bootstrapFile := path.Join(util.TestDir, "bootstrap-recoverytest", "kubeconfig")
+		// create an INVALID bootstrap kubeconfig file with an expired cert
+		err = util.CreateBootstrapKubeConfigWithCertAge(bootstrapFile, securePort, -1*time.Hour)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// run registration agent with an invalid bootstrap kubeconfig
+		go func() {
+			agentOptions := spoke.SpokeAgentOptions{
+				ClusterName:         spokeClusterName,
+				BootstrapKubeconfig: bootstrapFile,
+				HubKubeconfigSecret: hubKubeconfigSecret,
+				HubKubeconfigDir:    hubKubeconfigDir,
+			}
+			err := agentOptions.RunSpokeAgent(context.Background(), &controllercmd.ControllerContext{
+				KubeConfig:    spokeCfg,
+				EventRecorder: util.NewIntegrationTestEventRecorder("bootstrap-recoverytest"),
+			})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}()
+
+		// the spokecluster should not be created
+		retryToGetSpokeClusterTimes := 0
+		gomega.Eventually(func() int {
+			_, err = util.GetSpokeCluster(clusterClient, spokeClusterName)
+			gomega.Expect(err).To(gomega.HaveOccurred())
+			gomega.Expect(errors.IsNotFound(err)).Should(gomega.BeTrue())
+			retryToGetSpokeClusterTimes = retryToGetSpokeClusterTimes + 1
+			return retryToGetSpokeClusterTimes
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNumerically(">=", 3))
+
+		// the csr should not be created
+		retryToGetSpokeCSRTimes := 0
+		gomega.Eventually(func() int {
+			_, err := util.FindUnapprovedSpokeCSR(kubeClient, spokeClusterName)
+			gomega.Expect(err).To(gomega.HaveOccurred())
+			retryToGetSpokeCSRTimes = retryToGetSpokeCSRTimes + 1
+			return retryToGetSpokeCSRTimes
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNumerically(">=", 3))
+
+		// recover the invalid bootstrap kubeconfig file
+		err = util.CreateBootstrapKubeConfigWithCertAge(bootstrapFile, securePort, 24*time.Hour)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// the csr should be created after the bootstrap kubeconfig was recovered
+		gomega.Eventually(func() bool {
+			if _, err := util.FindUnapprovedSpokeCSR(kubeClient, spokeClusterName); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// the spoke cluster should be created after the bootstrap kubeconfig was recovered
+		gomega.Eventually(func() bool {
+			if _, err := util.GetSpokeCluster(clusterClient, spokeClusterName); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// simulate hub cluster admin accept the spoke cluster and approve the csr
+		err = util.AcceptSpokeCluster(clusterClient, spokeClusterName)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		err = util.ApproveSpokeClusterCSR(kubeClient, spokeClusterName, time.Hour*24)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// the hub kubeconfig secret should be filled after the csr is approved
+		gomega.Eventually(func() bool {
+			if _, err := util.GetFilledHubKubeConfigSecret(kubeClient, testNamespace, hubKubeconfigSecret); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// simulate k8s to mount the hub kubeconfig secret
+		err = util.MountHubKubeConfigs(kubeClient, hubKubeconfigDir, testNamespace, hubKubeconfigSecret)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// the spoke cluster should have joined condition finally
+		gomega.Eventually(func() bool {
+			spokeCluster, err := util.GetSpokeCluster(clusterClient, spokeClusterName)
+			if err != nil {
+				return false
+			}
+			joined := helpers.FindSpokeClusterCondition(spokeCluster.Status.Conditions, clusterv1.SpokeClusterConditionJoined)
+			if joined == nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+	})
+
+	ginkgo.It("agent recovery from invalid hub kubeconfig", func() {
+		var err error
+
+		spokeClusterName := "hubkubeconfig-recoverytest-spokecluster"
+
+		hubKubeconfigSecret := "hubkubeconfig-recoverytest-hub-kubeconfig-secret"
+		hubKubeconfigDir := path.Join(util.TestDir, "hubkubeconfig-recoverytest", "hub-kubeconfig")
+
+		// run registration agent
+		go func() {
+			agentOptions := spoke.SpokeAgentOptions{
+				ClusterName:         spokeClusterName,
+				BootstrapKubeconfig: bootstrapKubeConfigFile,
+				HubKubeconfigSecret: hubKubeconfigSecret,
+				HubKubeconfigDir:    hubKubeconfigDir,
+			}
+			err := agentOptions.RunSpokeAgent(context.Background(), &controllercmd.ControllerContext{
+				KubeConfig:    spokeCfg,
+				EventRecorder: util.NewIntegrationTestEventRecorder("hubkubeconfig-recoverytest"),
+			})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}()
+
+		// after bootstrap the spokecluster and csr should be created
+		gomega.Eventually(func() bool {
+			if _, err := util.GetSpokeCluster(clusterClient, spokeClusterName); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		var firstCSRName string
+		gomega.Eventually(func() bool {
+			csr, err := util.FindUnapprovedSpokeCSR(kubeClient, spokeClusterName)
+			if err != nil {
+				return false
+			}
+			firstCSRName = csr.Name
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// simulate hub cluster admin accept the spoke cluster
+		err = util.AcceptSpokeCluster(clusterClient, spokeClusterName)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// simulate hub cluster admin approve the csr with an INVALID hub config
+		err = util.ApproveSpokeClusterCSRWithExpiredCert(kubeClient, spokeClusterName)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// agent should bootstrap again due to the invalid hub config
+		var secondCSRName string
+		gomega.Eventually(func() bool {
+			csr, err := util.FindUnapprovedSpokeCSR(kubeClient, spokeClusterName)
+			if err != nil {
+				return false
+			}
+			secondCSRName = csr.Name
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// a new csr should be recreated
+		gomega.Expect(firstCSRName).ShouldNot(gomega.BeEquivalentTo(secondCSRName))
+
+		// approve the new csr
+		err = util.ApproveSpokeClusterCSR(kubeClient, spokeClusterName, time.Hour*24)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// the hub kubeconfig secret should be filled after the csr is approved
+		gomega.Eventually(func() bool {
+			if _, err := util.GetFilledHubKubeConfigSecret(kubeClient, testNamespace, hubKubeconfigSecret); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// simulate k8s to mount the hub kubeconfig secret
+		err = util.MountHubKubeConfigs(kubeClient, hubKubeconfigDir, testNamespace, hubKubeconfigSecret)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// the spoke cluster should have joined condition finally
+		gomega.Eventually(func() bool {
+			spokeCluster, err := util.GetSpokeCluster(clusterClient, spokeClusterName)
+			if err != nil {
+				return false
+			}
+			joined := helpers.FindSpokeClusterCondition(spokeCluster.Status.Conditions, clusterv1.SpokeClusterConditionJoined)
+			if joined == nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+	})
+})

test/integration/spokecluster_joining_test.go

@@ -0,0 +1,115 @@
+package integration_test
+
+import (
+	"context"
+	"path"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	clusterv1 "github.com/open-cluster-management/api/cluster/v1"
+	"github.com/open-cluster-management/registration/pkg/helpers"
+	"github.com/open-cluster-management/registration/pkg/spoke"
+	"github.com/open-cluster-management/registration/test/integration/util"
+
+	"github.com/openshift/library-go/pkg/controller/controllercmd"
+)
+
+var _ = ginkgo.Describe("Joining Process", func() {
+	ginkgo.It("spokecluster should join successfully", func() {
+		var err error
+
+		spokeClusterName := "joiningtest-spokecluster"
+		hubKubeconfigSecret := "joiningtest-hub-kubeconfig-secret"
+		hubKubeconfigDir := path.Join(util.TestDir, "joiningtest", "hub-kubeconfig")
+
+		// run registration agent
+		go func() {
+			agentOptions := spoke.SpokeAgentOptions{
+				ClusterName:         spokeClusterName,
+				BootstrapKubeconfig: bootstrapKubeConfigFile,
+				HubKubeconfigSecret: hubKubeconfigSecret,
+				HubKubeconfigDir:    hubKubeconfigDir,
+			}
+			err := agentOptions.RunSpokeAgent(context.Background(), &controllercmd.ControllerContext{
+				KubeConfig:    spokeCfg,
+				EventRecorder: util.NewIntegrationTestEventRecorder("joiningtest"),
+			})
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}()
+
+		// the spoke cluster and csr should be created after bootstrap
+		gomega.Eventually(func() bool {
+			if _, err := util.GetSpokeCluster(clusterClient, spokeClusterName); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		gomega.Eventually(func() bool {
+			if _, err := util.FindUnapprovedSpokeCSR(kubeClient, spokeClusterName); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// the spoke cluster should has finalizer that is added by hub controller
+		gomega.Eventually(func() bool {
+			spokeCluster, err := util.GetSpokeCluster(clusterClient, spokeClusterName)
+			if err != nil {
+				return false
+			}
+			if len(spokeCluster.Finalizers) != 1 ||
+				spokeCluster.Finalizers[0] != "cluster.open-cluster-management.io/api-resource-cleanup" {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// simulate hub cluster admin to accept the spokecluster and approve the csr
+		err = util.AcceptSpokeCluster(clusterClient, spokeClusterName)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		err = util.ApproveSpokeClusterCSR(kubeClient, spokeClusterName, time.Hour*24)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// the spoke cluster should have accepted condition after it is accepted
+		gomega.Eventually(func() bool {
+			spokeCluster, err := util.GetSpokeCluster(clusterClient, spokeClusterName)
+			if err != nil {
+				return false
+			}
+			accpeted := helpers.FindSpokeClusterCondition(spokeCluster.Status.Conditions, clusterv1.SpokeClusterConditionHubAccepted)
+			if accpeted == nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// the hub kubeconfig secret should be filled after the csr is approved
+		gomega.Eventually(func() bool {
+			if _, err := util.GetFilledHubKubeConfigSecret(kubeClient, testNamespace, hubKubeconfigSecret); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// simulate k8s to mount the hub kubeconfig secret
+		err = util.MountHubKubeConfigs(kubeClient, hubKubeconfigDir, testNamespace, hubKubeconfigSecret)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		// the spoke cluster should have joined condition finally
+		gomega.Eventually(func() bool {
+			spokeCluster, err := util.GetSpokeCluster(clusterClient, spokeClusterName)
+			if err != nil {
+				return false
+			}
+			joined := helpers.FindSpokeClusterCondition(spokeCluster.Status.Conditions, clusterv1.SpokeClusterConditionJoined)
+			if joined == nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+	})
+})

test/integration/util/util.go

@@ -0,0 +1,541 @@
+package util
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"math/big"
+	"net"
+	"os"
+	"path"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	clusterclientset "github.com/open-cluster-management/api/client/cluster/clientset/versioned"
+	clusterv1 "github.com/open-cluster-management/api/cluster/v1"
+	"github.com/openshift/library-go/pkg/operator/events"
+	certificates "k8s.io/api/certificates/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/keyutil"
+)
+
+const TestDir = "/tmp/registration-integration-test"
+
+var (
+	CertDir        = path.Join(TestDir, "server-certs")
+	CAFile         = path.Join(CertDir, "ca.crt")
+	CAKeyFile      = path.Join(CertDir, "ca.key")
+	ServerCertFile = path.Join(CertDir, "apiserver.crt")
+	ServerKeyFile  = path.Join(CertDir, "apiserver.key")
+)
+
+func CreateBootstrapKubeConfig(configFileName string, securePort int) error {
+	caData, err := ioutil.ReadFile(CAFile)
+	if err != nil {
+		return err
+	}
+
+	config := clientcmdapi.NewConfig()
+	config.Clusters["hub"] = &clientcmdapi.Cluster{
+		Server:                   fmt.Sprintf("https://127.0.0.1:%d", securePort),
+		CertificateAuthorityData: caData,
+	}
+	config.AuthInfos["bootstrap"] = &clientcmdapi.AuthInfo{
+		ClientCertificate: ServerCertFile,
+		ClientKey:         ServerKeyFile,
+	}
+	config.Contexts["bootstrap"] = &clientcmdapi.Context{
+		Cluster:  "hub",
+		AuthInfo: "bootstrap",
+	}
+	config.CurrentContext = "bootstrap"
+
+	return clientcmd.WriteToFile(*config, configFileName)
+}
+
+func CreateBootstrapKubeConfigWithCertAge(configFileName string, securePort int, certAge time.Duration) error {
+	caData, err := ioutil.ReadFile(CAFile)
+	if err != nil {
+		return err
+	}
+
+	certData, keyData, err := SignAPIServerCertKeyWithCA(certAge)
+	if err != nil {
+		return err
+	}
+
+	configDir := path.Dir(configFileName)
+	if _, err := os.Stat(configDir); os.IsNotExist(err) {
+		if err = os.MkdirAll(configDir, 0755); err != nil {
+			return err
+		}
+	}
+
+	if err := ioutil.WriteFile(path.Join(configDir, "bootstrap.crt"), certData, 0644); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(path.Join(configDir, "bootstrap.key"), keyData, 0644); err != nil {
+		return err
+	}
+
+	config := clientcmdapi.NewConfig()
+	config.Clusters["hub"] = &clientcmdapi.Cluster{
+		Server:                   fmt.Sprintf("https://127.0.0.1:%d", securePort),
+		CertificateAuthorityData: caData,
+	}
+	config.AuthInfos["bootstrap"] = &clientcmdapi.AuthInfo{
+		ClientCertificate: path.Join(configDir, "bootstrap.crt"),
+		ClientKey:         path.Join(configDir, "bootstrap.key"),
+	}
+	config.Contexts["bootstrap"] = &clientcmdapi.Context{
+		Cluster:  "hub",
+		AuthInfo: "bootstrap",
+	}
+	config.CurrentContext = "bootstrap"
+
+	return clientcmd.WriteToFile(*config, configFileName)
+}
+
+func CreateSpokeKubeConfig(restConfig *rest.Config, securePort int) (*rest.Config, error) {
+	spokeConfig := rest.CopyConfig(restConfig)
+	spokeConfig.Host = fmt.Sprintf("127.0.0.1:%d", securePort)
+
+	caData, err := ioutil.ReadFile(CAFile)
+	if err != nil {
+		return nil, err
+	}
+	spokeConfig.CAData = caData
+
+	certData, err := ioutil.ReadFile(ServerCertFile)
+	if err != nil {
+		return nil, err
+	}
+	spokeConfig.CertData = certData
+
+	keyData, err := ioutil.ReadFile(ServerKeyFile)
+	if err != nil {
+		return nil, err
+	}
+	spokeConfig.KeyData = keyData
+	return spokeConfig, nil
+}
+
+func GenerateSelfSignedCertKey() error {
+	if _, err := os.Stat(CertDir); os.IsNotExist(err) {
+		if err = os.MkdirAll(CertDir, 0755); err != nil {
+			return err
+		}
+	}
+
+	now := time.Now()
+	maxAge := time.Hour * 24
+
+	// generate ca cert and key
+	caKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return err
+	}
+
+	caTemplate := x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "127.0.0.1"},
+		NotBefore:             now.UTC(),
+		NotAfter:              now.Add(maxAge).UTC(),
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	caDERBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey)
+	if err != nil {
+		return err
+	}
+
+	caCertBuffer := bytes.Buffer{}
+	if err := pem.Encode(&caCertBuffer, &pem.Block{Type: certutil.CertificateBlockType, Bytes: caDERBytes}); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(CAFile, caCertBuffer.Bytes(), 0644); err != nil {
+		return err
+	}
+
+	caKeyBuffer := bytes.Buffer{}
+	if err := pem.Encode(
+		&caKeyBuffer, &pem.Block{Type: keyutil.RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(caKey)}); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(CAKeyFile, caKeyBuffer.Bytes(), 0644); err != nil {
+		return err
+	}
+
+	serverCert, serverKey, err := SignAPIServerCertKeyWithCA(maxAge)
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(ServerCertFile, serverCert, 0644); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(ServerKeyFile, serverKey, 0644); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func SignAPIServerCertKeyWithCA(maxAge time.Duration) ([]byte, []byte, error) {
+	now := time.Now()
+	caData, err := ioutil.ReadFile(CAFile)
+	if err != nil {
+		return nil, nil, err
+	}
+	caBlock, _ := pem.Decode(caData)
+	caCert, err := x509.ParseCertificate(caBlock.Bytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caKeyData, err := ioutil.ReadFile(CAKeyFile)
+	if err != nil {
+		return nil, nil, err
+	}
+	keyBlock, _ := pem.Decode(caKeyData)
+	caKey, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	serverKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	serverDERBytes, err := x509.CreateCertificate(
+		rand.Reader,
+		&x509.Certificate{
+			SerialNumber:          big.NewInt(1),
+			Subject:               pkix.Name{Organization: []string{"registration.integration.test"}, CommonName: "127.0.0.1"},
+			NotBefore:             now.UTC(),
+			NotAfter:              now.Add(maxAge).UTC(),
+			BasicConstraintsValid: false,
+			IsCA:                  false,
+			KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDataEncipherment,
+			ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+			IPAddresses:           []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("10.0.0.0")},
+			DNSNames: []string{
+				"kubernetes",
+				"kubernetes.default",
+				"kubernetes.default.svc",
+				"kubernetes.default.svc.cluster",
+				"kubernetes.default.svc.cluster.local",
+			},
+		},
+		caCert,
+		&serverKey.PublicKey,
+		caKey,
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+	certBuffer := bytes.Buffer{}
+	if err := pem.Encode(&certBuffer, &pem.Block{Type: certutil.CertificateBlockType, Bytes: serverDERBytes}); err != nil {
+		return nil, nil, err
+	}
+
+	keyBuffer := bytes.Buffer{}
+	if err := pem.Encode(
+		&keyBuffer, &pem.Block{Type: keyutil.RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(serverKey)}); err != nil {
+		return nil, nil, err
+	}
+	return certBuffer.Bytes(), keyBuffer.Bytes(), nil
+}
+
+func PrepareSpokeAgentNamespace(kubeClient kubernetes.Interface, namespace string) error {
+	_, err := kubeClient.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{})
+	if err == nil {
+		return nil
+	}
+	if !errors.IsNotFound(err) {
+		return err
+	}
+	ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
+	_, err = kubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
+	return err
+}
+
+func GetFilledHubKubeConfigSecret(kubeClient kubernetes.Interface, secretNamespace, secretName string) (*corev1.Secret, error) {
+	secret, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(context.TODO(), secretName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	if _, existed := secret.Data["cluster-name"]; !existed {
+		return nil, fmt.Errorf("cluster-name is not found\n")
+	}
+
+	if _, existed := secret.Data["agent-name"]; !existed {
+		return nil, fmt.Errorf("agent-name is not found\n")
+	}
+
+	if _, existed := secret.Data["kubeconfig"]; !existed {
+		return nil, fmt.Errorf("kubeconfig is not found\n")
+	}
+
+	if _, existed := secret.Data["tls.crt"]; !existed {
+		return nil, fmt.Errorf("tls.crt is not found\n")
+	}
+
+	if _, existed := secret.Data["tls.key"]; !existed {
+		return nil, fmt.Errorf("tls.key is not found\n")
+	}
+	return secret, nil
+}
+
+func MountHubKubeConfigs(kubeClient kubernetes.Interface, hubKubeConfigDir, secretNamespace, secretName string) error {
+	if _, err := os.Stat(hubKubeConfigDir); os.IsNotExist(err) {
+		if err = os.MkdirAll(hubKubeConfigDir, 0755); err != nil {
+			return err
+		}
+	}
+
+	secret, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(context.TODO(), secretName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(path.Join(hubKubeConfigDir, "cluster-name"), secret.Data["cluster-name"], 0644); err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(path.Join(hubKubeConfigDir, "agent-name"), secret.Data["agent-name"], 0644); err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(path.Join(hubKubeConfigDir, "tls.crt"), secret.Data["tls.crt"], 0644); err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(path.Join(hubKubeConfigDir, "tls.key"), secret.Data["tls.key"], 0644); err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(path.Join(hubKubeConfigDir, "kubeconfig"), secret.Data["kubeconfig"], 0644); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func FindUnapprovedSpokeCSR(kubeClient kubernetes.Interface, spokeClusterName string) (*certificates.CertificateSigningRequest, error) {
+	csrList, err := kubeClient.CertificatesV1beta1().CertificateSigningRequests().List(context.TODO(), metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("open-cluster-management.io/cluster-name=%s", spokeClusterName),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var unapproved *certificates.CertificateSigningRequest
+	for _, csr := range csrList.Items {
+		if len(csr.Status.Conditions) == 0 {
+			unapproved = csr.DeepCopy()
+			break
+		}
+	}
+
+	if unapproved == nil {
+		return nil, fmt.Errorf("failed to find unapproved csr for spoke cluster %q\n", spokeClusterName)
+	}
+
+	return unapproved, nil
+}
+
+func FindAutoApprovedSpokeCSR(kubeClient kubernetes.Interface, spokeClusterName string) (*certificates.CertificateSigningRequest, error) {
+	csrList, err := kubeClient.CertificatesV1beta1().CertificateSigningRequests().List(context.TODO(), metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("open-cluster-management.io/cluster-name=%s", spokeClusterName),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var autoApproved *certificates.CertificateSigningRequest
+	for _, csr := range csrList.Items {
+		if len(csr.Status.Conditions) == 0 {
+			continue
+		}
+		cond := csr.Status.Conditions[0]
+		if cond.Type == certificates.CertificateApproved &&
+			cond.Reason == "AutoApprovedByHubCSRApprovingController" {
+			autoApproved = csr.DeepCopy()
+			break
+		}
+	}
+
+	if autoApproved == nil {
+		return nil, fmt.Errorf("failed to find autoapproved csr for spoke cluster %q\n", spokeClusterName)
+	}
+
+	return autoApproved, nil
+}
+
+func ApproveSpokeClusterCSRWithExpiredCert(kubeClient kubernetes.Interface, spokeClusterName string) error {
+	now := time.Now()
+
+	csr, err := FindUnapprovedSpokeCSR(kubeClient, spokeClusterName)
+	if err != nil {
+		return err
+	}
+
+	return ApproveCSR(kubeClient, csr, now.UTC(), now.Add(-1*time.Hour).UTC())
+}
+
+func ApproveSpokeClusterCSR(kubeClient kubernetes.Interface, spokeClusterName string, certAge time.Duration) error {
+	now := time.Now()
+
+	csr, err := FindUnapprovedSpokeCSR(kubeClient, spokeClusterName)
+	if err != nil {
+		return err
+	}
+
+	return ApproveCSR(kubeClient, csr, now.UTC(), now.Add(certAge).UTC())
+}
+
+func ApproveCSR(kubeClient kubernetes.Interface, csr *certificates.CertificateSigningRequest, notBefore, notAfter time.Time) error {
+	block, _ := pem.Decode(csr.Spec.Request)
+	cr, err := x509.ParseCertificateRequest(block.Bytes)
+	if err != nil {
+		return err
+	}
+
+	caData, err := ioutil.ReadFile(CAFile)
+	if err != nil {
+		return err
+	}
+	caBlock, _ := pem.Decode(caData)
+	caCert, err := x509.ParseCertificate(caBlock.Bytes)
+	if err != nil {
+		return err
+	}
+
+	caKeyData, err := ioutil.ReadFile(CAKeyFile)
+	if err != nil {
+		return err
+	}
+	keyBlock, _ := pem.Decode(caKeyData)
+	caKey, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes)
+	if err != nil {
+		return err
+	}
+
+	certDERBytes, err := x509.CreateCertificate(
+		rand.Reader,
+		&x509.Certificate{
+			SerialNumber:          big.NewInt(1),
+			Subject:               pkix.Name{Organization: cr.Subject.Organization, CommonName: cr.Subject.CommonName},
+			NotBefore:             notBefore,
+			NotAfter:              notAfter,
+			BasicConstraintsValid: false,
+			IsCA:                  false,
+			KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+			ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		},
+		caCert,
+		cr.PublicKey,
+		caKey,
+	)
+	if err != nil {
+		return err
+	}
+
+	certBuffer := bytes.Buffer{}
+	if err := pem.Encode(&certBuffer, &pem.Block{Type: certutil.CertificateBlockType, Bytes: certDERBytes}); err != nil {
+		return err
+	}
+
+	// set cert
+	csr.Status = certificates.CertificateSigningRequestStatus{
+		Certificate: certBuffer.Bytes(),
+		Conditions:  []certificates.CertificateSigningRequestCondition{},
+	}
+	_, err = kubeClient.CertificatesV1beta1().CertificateSigningRequests().UpdateStatus(context.TODO(), csr, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	// approve the csr
+	approved, err := kubeClient.CertificatesV1beta1().CertificateSigningRequests().Get(context.TODO(), csr.Name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	approved.Status.Conditions = append(approved.Status.Conditions, certificates.CertificateSigningRequestCondition{
+		Type:           certificates.CertificateApproved,
+		Reason:         "Approved",
+		Message:        "CSR Approved.",
+		LastUpdateTime: metav1.Now(),
+	})
+	_, err = kubeClient.CertificatesV1beta1().CertificateSigningRequests().UpdateApproval(context.TODO(), approved, metav1.UpdateOptions{})
+	return err
+}
+
+func GetSpokeCluster(clusterClient clusterclientset.Interface, spokeClusterName string) (*clusterv1.SpokeCluster, error) {
+	spokeCluster, err := clusterClient.ClusterV1().SpokeClusters().Get(context.TODO(), spokeClusterName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return spokeCluster, nil
+}
+
+func AcceptSpokeCluster(clusterClient clusterclientset.Interface, spokeClusterName string) error {
+	spokeCluster, err := GetSpokeCluster(clusterClient, spokeClusterName)
+	if err != nil {
+		return err
+	}
+	spokeCluster.Spec.HubAcceptsClient = true
+	_, err = clusterClient.ClusterV1().SpokeClusters().Update(context.TODO(), spokeCluster, metav1.UpdateOptions{})
+	return err
+}
+
+func NewIntegrationTestEventRecorder(componet string) events.Recorder {
+	return &IntegrationTestEventRecorder{component: componet}
+}
+
+type IntegrationTestEventRecorder struct {
+	component string
+}
+
+func (r *IntegrationTestEventRecorder) ComponentName() string {
+	return r.component
+}
+
+func (r *IntegrationTestEventRecorder) ForComponent(c string) events.Recorder {
+	return &IntegrationTestEventRecorder{component: c}
+}
+
+func (r *IntegrationTestEventRecorder) WithComponentSuffix(suffix string) events.Recorder {
+	return r.ForComponent(fmt.Sprintf("%s-%s", r.ComponentName(), suffix))
+}
+
+func (r *IntegrationTestEventRecorder) Event(reason, message string) {
+	fmt.Fprintf(ginkgo.GinkgoWriter, "Event: [%s] %v: %v \n", r.component, reason, message)
+}
+
+func (r *IntegrationTestEventRecorder) Eventf(reason, messageFmt string, args ...interface{}) {
+	r.Event(reason, fmt.Sprintf(messageFmt, args...))
+}
+
+func (r *IntegrationTestEventRecorder) Warning(reason, message string) {
+	fmt.Fprintf(ginkgo.GinkgoWriter, "Warning: [%s] %v: %v \n", r.component, reason, message)
+}
+
+func (r *IntegrationTestEventRecorder) Warningf(reason, messageFmt string, args ...interface{}) {
+	r.Warning(reason, fmt.Sprintf(messageFmt, args...))
+}