diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..56bddcd --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,38 @@ +name: Tests + +on: + push: + +permissions: + contents: read + +env: + GO_VERSION: "1.22" + +jobs: + tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: ${{env.GO_VERSION}} + + - name: Install tools + run: | + go install github.com/onsi/ginkgo/v2/ginkgo + go get github.com/onsi/gomega/... + + # With Golang 1.22 we need to use the release-0.18 branch + go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.18 + + ENVTEST_BIN=$(setup-envtest use -p path) + sudo mkdir -p /usr/local/kubebuilder/bin + sudo cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin + + - name: Run tests + run: | + ginkgo run ./... diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 0037d51..69c4b0b 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -16,6 +16,14 @@ RUN curl -sL https://github.com/helm/chart-releaser/releases/download/v1.5.0/cha ENV CONTROLLER_GEN_VERSION v0.14.0 RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_GEN_VERSION} +# Tool to setup the envtest framework to run the controllers integration tests +# Note: With Golang 1.22 we need to use the release-0.18 branch +ENV SETUP_ENVTEST_VERSION release-0.18 +RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@${SETUP_ENVTEST_VERSION} && \ + ENVTEST_BIN=$(setup-envtest use -p path) && \ + mkdir -p /usr/local/kubebuilder/bin && \ + cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin + ENV GO111MODULE on ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS GITHUB_TOKEN ENV DAPPER_SOURCE /go/src/github.com/rancher/k3k/ diff --git a/go.mod b/go.mod index 73272bb..76e6f52 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,8 @@ replace ( require ( github.com/go-logr/zapr v1.3.0 + github.com/onsi/ginkgo/v2 v2.14.0 + github.com/onsi/gomega v1.30.0 github.com/prometheus/client_model v0.6.1 github.com/rancher/dynamiclistener v1.27.5 github.com/sirupsen/logrus v1.9.3 @@ -58,6 +60,7 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -65,6 +68,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -110,6 +114,7 @@ require ( golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.16.1 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect diff --git a/go.sum b/go.sum index 10b86bc..6a35960 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,9 @@ github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= @@ -134,6 +137,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -209,6 +213,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -301,6 +306,7 @@ golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= diff --git a/pkg/controller/clusterset/clusterset_suite_test.go b/pkg/controller/clusterset/clusterset_suite_test.go new file mode 100644 index 0000000..3afec6f --- /dev/null +++ b/pkg/controller/clusterset/clusterset_suite_test.go @@ -0,0 +1,88 @@ +package clusterset_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + "github.com/rancher/k3k/pkg/controller/clusterset" + "github.com/rancher/k3k/pkg/log" + + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ClusterSet Controller Suite") +} + +var ( + testEnv *envtest.Environment + k8sClient client.Client + ctx context.Context + cancel context.CancelFunc +) + +var _ = BeforeSuite(func() { + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")}, + ErrorIfCRDPathMissing: true, + } + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + + scheme := buildScheme() + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + + ctx, cancel = context.WithCancel(context.Background()) + nopLogger := &log.Logger{SugaredLogger: zap.NewNop().Sugar()} + + err = clusterset.Add(ctx, mgr, "", nopLogger) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred(), "failed to run manager") + }() +}) + +var _ = AfterSuite(func() { + cancel() + + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +func buildScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + + err := corev1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = appsv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = networkingv1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = v1alpha1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + return scheme +} diff --git a/pkg/controller/clusterset/clusterset_test.go b/pkg/controller/clusterset/clusterset_test.go new file mode 100644 index 0000000..c1b202b --- /dev/null +++ b/pkg/controller/clusterset/clusterset_test.go @@ -0,0 +1,129 @@ +package clusterset_test + +import ( + "context" + "time" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + + k3kcontroller "github.com/rancher/k3k/pkg/controller" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ClusterSet Controller", func() { + + Context("creating a ClusterSet", func() { + + var ( + namespace string + ) + + BeforeEach(func() { + createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}} + err := k8sClient.Create(context.Background(), createdNS) + Expect(err).To(Not(HaveOccurred())) + namespace = createdNS.Name + }) + + When("created with a default spec", func() { + It("should create a NetworkPolicy", func() { + clusterSet := &v1alpha1.ClusterSet{ + ObjectMeta: v1.ObjectMeta{ + GenerateName: "clusterset-", + Namespace: namespace, + }, + } + + err := k8sClient.Create(ctx, clusterSet) + Expect(err).To(Not(HaveOccurred())) + + // look for network policies etc + clusterSetNetworkPolicy := &networkingv1.NetworkPolicy{} + + Eventually(func() bool { + deployKey := types.NamespacedName{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: namespace, + } + err := k8sClient.Get(ctx, deployKey, clusterSetNetworkPolicy) + return err == nil + }, time.Minute, time.Second).Should(BeTrue()) + + spec := clusterSetNetworkPolicy.Spec + Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeEgress)) + Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeIngress)) + + // ingress should allow everything + Expect(spec.Ingress).To(ConsistOf(networkingv1.NetworkPolicyIngressRule{})) + + // egress should contains some rules + Expect(spec.Egress).To(HaveLen(1)) + + // allow networking to all external IPs + ipBlockRule := networkingv1.NetworkPolicyPeer{ + IPBlock: &networkingv1.IPBlock{CIDR: "0.0.0.0/0"}, + } + + // allow networking in the same namespace + clusterSetNamespaceRule := networkingv1.NetworkPolicyPeer{ + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"kubernetes.io/metadata.name": namespace}, + }, + } + + // allow networking to the "kube-dns" pod in the "kube-system" namespace + kubeDNSRule := networkingv1.NetworkPolicyPeer{ + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"k8s-app": "kube-dns"}, + }, + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"}, + }, + } + + Expect(spec.Egress[0].To).To(ContainElements( + ipBlockRule, clusterSetNamespaceRule, kubeDNSRule, + )) + }) + }) + + When("created with DisableNetworkPolicy", func() { + It("should not create a NetworkPolicy", func() { + clusterSet := &v1alpha1.ClusterSet{ + ObjectMeta: v1.ObjectMeta{ + GenerateName: "clusterset-", + Namespace: namespace, + }, + Spec: v1alpha1.ClusterSetSpec{ + DisableNetworkPolicy: true, + }, + } + + err := k8sClient.Create(ctx, clusterSet) + Expect(err).To(Not(HaveOccurred())) + + // wait for a bit for the network policy, but it should not be created + Eventually(func() bool { + deployKey := types.NamespacedName{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: namespace, + } + err := k8sClient.Get(ctx, deployKey, &networkingv1.NetworkPolicy{}) + return apierrors.IsNotFound(err) + }). + MustPassRepeatedly(5). + WithTimeout(time.Second * 10). + WithPolling(time.Second). + Should(BeTrue()) + }) + }) + }) +})