diff --git a/.gitignore b/.gitignore index 99073c0f1..873b13abd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Binaries for programs and plugins registration dev-kubeconfig +e2e-kubeconfig integration.test *.exe *.dll diff --git a/Dockerfile b/Dockerfile index aea4cd884..5633422c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,9 @@ FROM docker.io/openshift/origin-release:golang-1.13 AS builder WORKDIR /go/src/github.com/open-cluster-management/registration COPY . . ENV GO_PACKAGE github.com/open-cluster-management/registration + +RUN curl -O -L https://github.com/kubernetes-sigs/kustomize/releases/download/v2.0.0/kustomize_2.0.0_linux_amd64; mv kustomize_2.0.0_linux_amd64 kustomize; chmod u+x kustomize; mv kustomize /usr/bin/kustomize + RUN make build --warn-undefined-variables FROM registry.access.redhat.com/ubi8/ubi-minimal:8.1-398 diff --git a/Makefile b/Makefile index de41e66e3..6cbbb91f6 100644 --- a/Makefile +++ b/Makefile @@ -10,19 +10,22 @@ include $(addprefix ./vendor/github.com/openshift/build-machinery-go/make/, \ targets/openshift/bindata.mk \ ) -IMAGE_REGISTRY?=quay.io +IMAGE_REGISTRY?=quay.io/open-cluster-management IMAGE_TAG?=latest -IMAGE_NAME?=$(IMAGE_REGISTRY)/open-cluster-management/registration:$(IMAGE_TAG) +IMAGE_NAME?=$(IMAGE_REGISTRY)/registration:$(IMAGE_TAG) KUBECONFIG ?= ./.kubeconfig +KUBECTL?=kubectl $(call add-bindata,spokecluster,./pkg/hub/spokecluster/manifests/...,bindata,bindata,./pkg/hub/spokecluster/bindata/bindata.go) +$(call add-bindata,spokecluster-e2e,./deploy/spoke/...,bindata,bindata,./test/e2e/bindata/bindata.go) + # This will call a macro called "build-image" which will generate image specific targets based on the parameters: # $0 - macro name # $1 - target suffix # $2 - Dockerfile path # $3 - context directory for image build # It will generate target "image-$(1)" for builing the image an binding it as a prerequisite to target "images". -$(call build-image,registration,$(IMAGE_REGISTRY)/open-cluster-management/registration,./Dockerfile,.) +$(call build-image,registration,$(IMAGE_REGISTRY)/registration,./Dockerfile,.) clean: @@ -40,8 +43,14 @@ cluster-ip: bootstrap-secret: cluster-ip cp $(KUBECONFIG) dev-kubeconfig - kubectl config set clusters.kind-kind.server https://$(CLUSTER_IP) --kubeconfig dev-kubeconfig - kubectl create secret generic bootstrap-secret --from-file=kubeconfig=dev-kubeconfig -n open-cluster-management + $(KUBECTL) config set clusters.kind-kind.server https://$(CLUSTER_IP) --kubeconfig dev-kubeconfig + $(KUBECTL) create secret generic bootstrap-secret --from-file=kubeconfig=dev-kubeconfig -n open-cluster-management + +e2e-bootstrap-secret: cluster-ip + cp $(KUBECONFIG) e2e-kubeconfig + $(KUBECTL) config set clusters.kind-kind.server https://$(CLUSTER_IP) --kubeconfig e2e-kubeconfig + $(KUBECTL) delete secret e2e-bootstrap-secret -n open-cluster-management --ignore-not-found + $(KUBECTL) create secret generic e2e-bootstrap-secret --from-file=kubeconfig=e2e-kubeconfig -n open-cluster-management deploy-spoke: cp deploy/spoke/kustomization.yaml deploy/spoke/kustomization.yaml.tmp @@ -49,9 +58,12 @@ deploy-spoke: kustomize build deploy/spoke | kubectl apply -f - mv deploy/spoke/kustomization.yaml.tmp deploy/spoke/kustomization.yaml +deploy-all: deploy-hub bootstrap-secret deploy-spoke + # test-e2e target is currently a NOP that deploys the hub and self-joins the # hosting cluster to itself as a spoke; it will be used to prototype e2e in ci. -test-e2e: deploy-hub bootstrap-secret deploy-spoke +test-e2e: deploy-hub e2e-bootstrap-secret + go test ./test/e2e -v -ginkgo.v GO_TEST_PACKAGES :=./pkg/... ./cmd/... diff --git a/deploy/spoke/deployment.yaml b/deploy/spoke/deployment.yaml index 6ddc0d46a..a13fbb571 100644 --- a/deploy/spoke/deployment.yaml +++ b/deploy/spoke/deployment.yaml @@ -22,9 +22,9 @@ spec: args: - "/registration" - "agent" - - "--cluster-name=spokecluster1" + - "--cluster-name=local-development" - "--bootstrap-kubeconfig=/spoke/bootstrap/kubeconfig" - - "--spoke-external-server-url=https://127.0.0.1" + - "--spoke-external-server-urls=https://127.0.0.1" volumeMounts: - name: bootstrap-secret mountPath: "/spoke/bootstrap" diff --git a/go.sum b/go.sum index 90677d930..362fe3ee5 100644 --- a/go.sum +++ b/go.sum @@ -295,6 +295,7 @@ github.com/openshift/api v0.0.0-20200326160804-ecb9283fe820 h1:pEmlKM0gcAPwPEOUt github.com/openshift/api v0.0.0-20200326160804-ecb9283fe820/go.mod h1:RKMJ5CBnljLfnej+BJ/xnOWc3kZDvJUaIAEq2oKSPtE= github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160 h1:V4E6yt4XWiBEPKnJbs/E8pgUq9AjZqzQfsL3eeT84Qs= github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc= +github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0 h1:kMiuiZXH1GdfbiMwsuAQOqGaMxlo9NCUk0wT4XAdfNM= github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0/go.mod h1:uUQ4LClRO+fg5MF/P6QxjMCb1C9f7Oh4RKepftDnEJE= github.com/openshift/library-go v0.0.0-20200401114229-ffab8c6e83a9 h1:aEcgb9sKGmHK2IP9IcyUL11XYle0XNVnrG4871MXXIw= github.com/openshift/library-go v0.0.0-20200401114229-ffab8c6e83a9/go.mod h1:CfydoH0B+RYs22uQZQ36A1mz5m5zhucpMGh8t5s71v4= diff --git a/pkg/helpers/helpers.go b/pkg/helpers/helpers.go index e218e987c..97cdaf902 100644 --- a/pkg/helpers/helpers.go +++ b/pkg/helpers/helpers.go @@ -7,6 +7,7 @@ import ( spokeclusterclientset "github.com/open-cluster-management/api/client/cluster/clientset/versioned" spokeclusterv1 "github.com/open-cluster-management/api/cluster/v1" + certificatesv1beta1 "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -96,6 +97,19 @@ func UpdateSpokeClusterConditionFn(cond spokeclusterv1.StatusCondition) UpdateSp } } +// Check whether a CSR is in terminal state +func IsCSRInTerminalState(status *certificatesv1beta1.CertificateSigningRequestStatus) bool { + for _, c := range status.Conditions { + if c.Type == certificatesv1beta1.CertificateApproved { + return true + } + if c.Type == certificatesv1beta1.CertificateDenied { + return true + } + } + return false +} + // IsValidHTTPSURL validate whether a URL is https URL func IsValidHTTPSURL(serverURL string) bool { if serverURL == "" { diff --git a/pkg/hub/csr/controller.go b/pkg/hub/csr/controller.go index 5f0a9925e..4e06a6fbf 100644 --- a/pkg/hub/csr/controller.go +++ b/pkg/hub/csr/controller.go @@ -17,6 +17,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/klog" + + "github.com/open-cluster-management/registration/pkg/helpers" ) const ( @@ -57,7 +59,7 @@ func (c *csrApprovingController) sync(ctx context.Context, syncCtx factory.SyncC } // Current csr is in terminal state, do nothing. - if isCSRInTerminalState(&csr.Status) { + if helpers.IsCSRInTerminalState(&csr.Status) { return nil } @@ -163,16 +165,3 @@ func isSpokeClusterClientCertRenewal(csr *certificatesv1beta1.CertificateSigning return csr.Spec.Username == x509cr.Subject.CommonName } - -// Check whether a CSR is in terminal state -func isCSRInTerminalState(status *certificatesv1beta1.CertificateSigningRequestStatus) bool { - for _, c := range status.Conditions { - if c.Type == certificatesv1beta1.CertificateApproved { - return true - } - if c.Type == certificatesv1beta1.CertificateDenied { - return true - } - } - return false -} diff --git a/test/e2e/bindata/bindata.go b/test/e2e/bindata/bindata.go new file mode 100644 index 000000000..963418432 --- /dev/null +++ b/test/e2e/bindata/bindata.go @@ -0,0 +1,420 @@ +// Code generated by go-bindata. +// sources: +// deploy/spoke/clusterrole_binding.yaml +// deploy/spoke/deployment.yaml +// deploy/spoke/kustomization.yaml +// deploy/spoke/namespace.yaml +// deploy/spoke/secret.yaml +// deploy/spoke/service_account.yaml +// DO NOT EDIT! + +package bindata + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +type asset struct { + bytes []byte + info os.FileInfo +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +func (fi bindataFileInfo) Name() string { + return fi.name +} +func (fi bindataFileInfo) Size() int64 { + return fi.size +} +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} +func (fi bindataFileInfo) IsDir() bool { + return false +} +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var _deploySpokeClusterrole_bindingYaml = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:open-cluster-management:spoke +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: spoke-agent-sa + namespace: open-cluster-management + `) + +func deploySpokeClusterrole_bindingYamlBytes() ([]byte, error) { + return _deploySpokeClusterrole_bindingYaml, nil +} + +func deploySpokeClusterrole_bindingYaml() (*asset, error) { + bytes, err := deploySpokeClusterrole_bindingYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "deploy/spoke/clusterrole_binding.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _deploySpokeDeploymentYaml = []byte(`kind: Deployment +apiVersion: apps/v1 +metadata: + name: spoke-agent + labels: + app: spoke-agent +spec: + replicas: 1 + selector: + matchLabels: + app: spoke-agent + template: + metadata: + labels: + app: spoke-agent + spec: + serviceAccountName: spoke-agent-sa + containers: + - name: spoke-agent + image: quay.io/open-cluster-management/registration:latest + imagePullPolicy: IfNotPresent + args: + - "/registration" + - "agent" + - "--cluster-name=local-development" + - "--bootstrap-kubeconfig=/spoke/bootstrap/kubeconfig" + - "--spoke-external-server-urls=https://127.0.0.1" + volumeMounts: + - name: bootstrap-secret + mountPath: "/spoke/bootstrap" + readOnly: true + - name: hub-kubeconfig-secret + mountPath: "/spoke/hub-kubeconfig" + readOnly: true + volumes: + - name: bootstrap-secret + secret: + secretName: bootstrap-secret + - name: hub-kubeconfig-secret + secret: + secretName: hub-kubeconfig-secret +`) + +func deploySpokeDeploymentYamlBytes() ([]byte, error) { + return _deploySpokeDeploymentYaml, nil +} + +func deploySpokeDeploymentYaml() (*asset, error) { + bytes, err := deploySpokeDeploymentYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "deploy/spoke/deployment.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _deploySpokeKustomizationYaml = []byte(` +# Adds namespace to all resources. +namespace: open-cluster-management + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +#namePrefix: multicloud- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +# Each entry in this list must resolve to an existing +# resource definition in YAML. These are the resource +# files that kustomize reads, modifies and emits as a +# YAML string, with resources separated by document +# markers ("---"). +# +# General rule here is anything deployed by OLM bundles should go here as well, +# this is used in "make deploy" for developers and should mimic what OLM deploys +# for you. CRDs are an exception to this as we don't want to have to list them all +# here. These are deployed via a "make install" dependency. + +resources: +- ./namespace.yaml +- ./service_account.yaml +- ./clusterrole_binding.yaml +- ./deployment.yaml +- ./secret.yaml + +images: +- name: quay.io/open-cluster-management/registration:latest + newName: quay.io/open-cluster-management/registration + newTag: latest +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +`) + +func deploySpokeKustomizationYamlBytes() ([]byte, error) { + return _deploySpokeKustomizationYaml, nil +} + +func deploySpokeKustomizationYaml() (*asset, error) { + bytes, err := deploySpokeKustomizationYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "deploy/spoke/kustomization.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _deploySpokeNamespaceYaml = []byte(`apiVersion: v1 +kind: Namespace +metadata: + name: open-cluster-management +`) + +func deploySpokeNamespaceYamlBytes() ([]byte, error) { + return _deploySpokeNamespaceYaml, nil +} + +func deploySpokeNamespaceYaml() (*asset, error) { + bytes, err := deploySpokeNamespaceYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "deploy/spoke/namespace.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _deploySpokeSecretYaml = []byte(`apiVersion: v1 +kind: Secret +metadata: + name: hub-kubeconfig-secret +type: Opaque +data: + placeholder: YWRtaW4= +`) + +func deploySpokeSecretYamlBytes() ([]byte, error) { + return _deploySpokeSecretYaml, nil +} + +func deploySpokeSecretYaml() (*asset, error) { + bytes, err := deploySpokeSecretYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "deploy/spoke/secret.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _deploySpokeService_accountYaml = []byte(`apiVersion: v1 +kind: ServiceAccount +metadata: + name: spoke-agent-sa +`) + +func deploySpokeService_accountYamlBytes() ([]byte, error) { + return _deploySpokeService_accountYaml, nil +} + +func deploySpokeService_accountYaml() (*asset, error) { + bytes, err := deploySpokeService_accountYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "deploy/spoke/service_account.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, fmt.Errorf("Asset %s not found", name) +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, fmt.Errorf("AssetInfo %s not found", name) +} + +// AssetNames returns the names of the assets. +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "deploy/spoke/clusterrole_binding.yaml": deploySpokeClusterrole_bindingYaml, + "deploy/spoke/deployment.yaml": deploySpokeDeploymentYaml, + "deploy/spoke/kustomization.yaml": deploySpokeKustomizationYaml, + "deploy/spoke/namespace.yaml": deploySpokeNamespaceYaml, + "deploy/spoke/secret.yaml": deploySpokeSecretYaml, + "deploy/spoke/service_account.yaml": deploySpokeService_accountYaml, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + } + } + if node.Func != nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} + +var _bintree = &bintree{nil, map[string]*bintree{ + "deploy": {nil, map[string]*bintree{ + "spoke": {nil, map[string]*bintree{ + "clusterrole_binding.yaml": {deploySpokeClusterrole_bindingYaml, map[string]*bintree{}}, + "deployment.yaml": {deploySpokeDeploymentYaml, map[string]*bintree{}}, + "kustomization.yaml": {deploySpokeKustomizationYaml, map[string]*bintree{}}, + "namespace.yaml": {deploySpokeNamespaceYaml, map[string]*bintree{}}, + "secret.yaml": {deploySpokeSecretYaml, map[string]*bintree{}}, + "service_account.yaml": {deploySpokeService_accountYaml, map[string]*bintree{}}, + }}, + }}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go new file mode 100644 index 000000000..ff1a3f945 --- /dev/null +++ b/test/e2e/e2e_suite_test.go @@ -0,0 +1,61 @@ +package e2e + +import ( + "os" + "testing" + + ginkgo "github.com/onsi/ginkgo" + gomega "github.com/onsi/gomega" + + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + clusterv1client "github.com/open-cluster-management/api/client/cluster/clientset/versioned" +) + +func TestE2E(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "E2E suite") +} + +var ( + hubClient kubernetes.Interface + hubDynamicClient dynamic.Interface + clusterClient clusterv1client.Interface + imageRegistry string +) + +var _ = ginkgo.BeforeSuite(func() { + logf.SetLogger(zap.LoggerTo(ginkgo.GinkgoWriter, true)) + kubeconfig := os.Getenv("KUBECONFIG") + imageRegistry = os.Getenv("IMAGE_REGISTRY") + if imageRegistry == "" { + imageRegistry = "quay.io/open-cluster-management" + } + + err := func() error { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return err + } + + hubClient, err = kubernetes.NewForConfig(config) + if err != nil { + return err + } + + hubDynamicClient, err = dynamic.NewForConfig(config) + if err != nil { + return err + } + + clusterClient, err = clusterv1client.NewForConfig(config) + + return err + }() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) +}) diff --git a/test/e2e/loopback_test.go b/test/e2e/loopback_test.go new file mode 100644 index 000000000..48178f2cb --- /dev/null +++ b/test/e2e/loopback_test.go @@ -0,0 +1,343 @@ +package e2e + +import ( + "bytes" + "context" + goerrors "errors" + "fmt" + "io/ioutil" + "reflect" + "time" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + + clusterv1 "github.com/open-cluster-management/api/cluster/v1" + certificatesv1beta1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/apimachinery/pkg/runtime/serializer/streaming" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + + "github.com/open-cluster-management/registration/pkg/helpers" + "github.com/open-cluster-management/registration/test/e2e/bindata" +) + +var spokeNamespace string = "" + +var _ = ginkgo.Describe("Loopback registration [development]", func() { + ginkgo.It("Should register the hub as a spoke", func() { + var ( + err error + suffix = rand.String(6) + nsName = fmt.Sprintf("loopback-spoke-%v", suffix) + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + }, + } + ) + ginkgo.By(fmt.Sprintf("Deploying the spoke agent using suffix=%q ns=%q", suffix, nsName)) + err = wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + ns, err = hubClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) + if err != nil { + return false, err + } + + return true, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + // This test expects a bootstrap secret to exist in open-cluster-management/e2e-bootstrap-secret + e2eBootstrapSecret, err := hubClient.CoreV1().Secrets("open-cluster-management").Get(context.TODO(), "e2e-bootstrap-secret", metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + bootstrapSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: "bootstrap-secret", + }, + } + bootstrapSecret.Data = e2eBootstrapSecret.Data + err = wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + _, err = hubClient.CoreV1().Secrets(nsName).Create(context.TODO(), bootstrapSecret, metav1.CreateOptions{}) + if err != nil { + return false, err + } + + return true, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + var ( + crb *unstructured.Unstructured + crbResource = schema.GroupVersionResource{ + Group: "rbac.authorization.k8s.io", + Version: "v1", + Resource: "clusterrolebindings", + } + ) + crb, err = spokeCRB(nsName, suffix) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + _, err = hubDynamicClient.Resource(crbResource).Create(context.TODO(), crb, metav1.CreateOptions{}) + if err != nil { + return false, err + } + + return true, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + var ( + deployment *unstructured.Unstructured + deploymentResource = schema.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + } + ) + clusterName := fmt.Sprintf("loopback-e2e-%v", suffix) + deployment, err = spokeDeployment(nsName, clusterName, imageRegistry) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + _, err = hubDynamicClient.Resource(deploymentResource).Namespace(nsName).Create(context.TODO(), deployment, metav1.CreateOptions{}) + if err != nil { + return false, err + } + + return true, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: "hub-kubeconfig-secret", + }, + Data: map[string][]byte{ + "placeholder": []byte("YWRtaW4="), + }, + } + err = wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + _, err = hubClient.CoreV1().Secrets(nsName).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + return false, err + } + + return true, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: "spoke-agent-sa", + }, + } + err = wait.Poll(1*time.Second, 5*time.Second, func() (bool, error) { + var err error + _, err = hubClient.CoreV1().ServiceAccounts(nsName).Create(context.TODO(), sa, metav1.CreateOptions{}) + if err != nil { + return false, err + } + + return true, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + var ( + csrs *certificatesv1beta1.CertificateSigningRequestList + csrClient = hubClient.CertificatesV1beta1().CertificateSigningRequests() + ) + + ginkgo.By(fmt.Sprintf("Waiting for the CSR for cluster %q to exist", clusterName)) + err = wait.Poll(1*time.Second, 90*time.Second, func() (bool, error) { + var err error + csrs, err = csrClient.List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("open-cluster-management.io/cluster-name = %v", clusterName), + }) + if err != nil { + return false, err + } + + if len(csrs.Items) >= 1 { + return true, nil + } + + return false, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("Approving all pending CSRs") + var csr *certificatesv1beta1.CertificateSigningRequest + for i := range csrs.Items { + csr = &csrs.Items[i] + csr, err = csrClient.Get(context.TODO(), csr.Name, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + if helpers.IsCSRInTerminalState(&csr.Status) { + continue + } + + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1beta1.CertificateSigningRequestCondition{ + Type: certificatesv1beta1.CertificateApproved, + Reason: "Approved by E2E", + Message: "Approved as part of Loopback e2e", + }) + + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + _, err := csrClient.UpdateApproval(context.TODO(), csr, metav1.UpdateOptions{}) + return err + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + } + + var ( + spoke *clusterv1.SpokeCluster + spokeClusters = clusterClient.ClusterV1().SpokeClusters() + ) + + ginkgo.By(fmt.Sprintf("Waiting for SpokeCluster %q to exist", clusterName)) + err = retry.OnError(retry.DefaultRetry, errors.IsNotFound, func() error { + var err error + spoke, err = spokeClusters.Get(context.TODO(), clusterName, metav1.GetOptions{}) + return err + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + gomega.Expect(spoke.Spec.HubAcceptsClient).To(gomega.Equal(false)) + + ginkgo.By(fmt.Sprintf("Accepting SpokeCluster %q", clusterName)) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + var err error + spoke, err = spokeClusters.Get(context.TODO(), spoke.Name, metav1.GetOptions{}) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + spoke.Spec.HubAcceptsClient = true + spoke, err = spokeClusters.Update(context.TODO(), spoke, metav1.UpdateOptions{}) + return err + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + + ginkgo.By("Waiting for SpokeCluster to have HubAccepted=true") + err = wait.Poll(1*time.Second, 10*time.Second, func() (bool, error) { + var err error + spoke, err := spokeClusters.Get(context.TODO(), clusterName, metav1.GetOptions{}) + if err != nil { + return false, err + } + + condition := helpers.FindSpokeClusterCondition(spoke.Status.Conditions, "HubAcceptedSpoke") + if condition == nil { + return false, nil + } + + if helpers.IsConditionTrue(condition) { + return true, nil + } + + return false, nil + }) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + }) +}) + +func assetToUnstructured(name string) (*unstructured.Unstructured, error) { + yamlDecoder := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + raw := bindata.MustAsset(name) + reader := json.YAMLFramer.NewFrameReader(ioutil.NopCloser(bytes.NewReader(raw))) + d := streaming.NewDecoder(reader, yamlDecoder) + obj, _, err := d.Decode(nil, nil) + if err != nil { + return nil, err + } + + switch t := obj.(type) { + case *unstructured.Unstructured: + return t, nil + default: + return nil, fmt.Errorf("failed to convert object, unexpected type %s", reflect.TypeOf(obj)) + } +} + +func spokeCRB(nsName, suffix string) (*unstructured.Unstructured, error) { + crb, err := assetToUnstructured("deploy/spoke/clusterrole_binding.yaml") + if err != nil { + return nil, err + } + + name := crb.GetName() + name = fmt.Sprintf("%v-%v", name, suffix) + crb.SetName(name) + + subjects, found, err := unstructured.NestedSlice(crb.Object, "subjects") + if err != nil { + return nil, err + } + if !found { + return nil, goerrors.New("couldn't find CRB subjects") + } + + err = unstructured.SetNestedField(subjects[0].(map[string]interface{}), nsName, "namespace") + if err != nil { + return nil, err + } + + err = unstructured.SetNestedField(crb.Object, subjects, "subjects") + if err != nil { + return nil, err + } + + return crb, nil +} + +func spokeDeployment(nsName, clusterName, imageRegistry string) (*unstructured.Unstructured, error) { + deployment, err := assetToUnstructured("deploy/spoke/deployment.yaml") + if err != nil { + return nil, err + } + err = unstructured.SetNestedField(deployment.Object, nsName, "meta", "namespace") + if err != nil { + return nil, err + } + + containers, found, err := unstructured.NestedSlice(deployment.Object, "spec", "template", "spec", "containers") + if err != nil || !found || containers == nil { + return nil, fmt.Errorf("deployment containers not found or error in spec: %v", err) + } + + image := fmt.Sprintf("%v/registration:latest", imageRegistry) + if err := unstructured.SetNestedField(containers[0].(map[string]interface{}), image, "image"); err != nil { + return nil, err + } + + args, found, err := unstructured.NestedSlice(containers[0].(map[string]interface{}), "args") + if err != nil || !found || args == nil { + return nil, fmt.Errorf("container args not found or error in spec: %v", err) + } + + clusterNameArg := fmt.Sprintf("--cluster-name=%v", clusterName) + args[2] = clusterNameArg + + if err := unstructured.SetNestedField(containers[0].(map[string]interface{}), args, "args"); err != nil { + return nil, err + } + + if err := unstructured.SetNestedField(deployment.Object, containers, "spec", "template", "spec", "containers"); err != nil { + return nil, err + } + + return deployment, nil +} diff --git a/vendor/k8s.io/apimachinery/pkg/runtime/serializer/yaml/yaml.go b/vendor/k8s.io/apimachinery/pkg/runtime/serializer/yaml/yaml.go new file mode 100644 index 000000000..2fdd1d43d --- /dev/null +++ b/vendor/k8s.io/apimachinery/pkg/runtime/serializer/yaml/yaml.go @@ -0,0 +1,46 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package yaml + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// yamlSerializer converts YAML passed to the Decoder methods to JSON. +type yamlSerializer struct { + // the nested serializer + runtime.Serializer +} + +// yamlSerializer implements Serializer +var _ runtime.Serializer = yamlSerializer{} + +// NewDecodingSerializer adds YAML decoding support to a serializer that supports JSON. +func NewDecodingSerializer(jsonSerializer runtime.Serializer) runtime.Serializer { + return &yamlSerializer{jsonSerializer} +} + +func (c yamlSerializer) Decode(data []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { + out, err := yaml.ToJSON(data) + if err != nil { + return nil, nil, err + } + data = out + return c.Serializer.Decode(data, gvk, into) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 67a4bfe19..e75fb0ad8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -469,6 +469,7 @@ k8s.io/apimachinery/pkg/runtime/serializer/protobuf k8s.io/apimachinery/pkg/runtime/serializer/recognizer k8s.io/apimachinery/pkg/runtime/serializer/streaming k8s.io/apimachinery/pkg/runtime/serializer/versioning +k8s.io/apimachinery/pkg/runtime/serializer/yaml k8s.io/apimachinery/pkg/selection k8s.io/apimachinery/pkg/types k8s.io/apimachinery/pkg/util/cache