mirror of
https://github.com/rancher/k3k.git
synced 2026-08-18 20:07:06 +00:00
Added envtest integration tests for ClusterSet (#143)
* init tests * added clusterset tests * added github action * updated Dapper with envtest bins
This commit is contained in:
Enrico Candino
@@ -0,0 +1,38 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.22"
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{env.GO_VERSION}}
|
||||
|
||||
- name: Install tools
|
||||
run: |
|
||||
go install github.com/onsi/ginkgo/v2/ginkgo
|
||||
go get github.com/onsi/gomega/...
|
||||
|
||||
# With Golang 1.22 we need to use the release-0.18 branch
|
||||
go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.18
|
||||
|
||||
ENVTEST_BIN=$(setup-envtest use -p path)
|
||||
sudo mkdir -p /usr/local/kubebuilder/bin
|
||||
sudo cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
ginkgo run ./...
|
||||
@@ -16,6 +16,14 @@ RUN curl -sL https://github.com/helm/chart-releaser/releases/download/v1.5.0/cha
|
||||
ENV CONTROLLER_GEN_VERSION v0.14.0
|
||||
RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_GEN_VERSION}
|
||||
|
||||
# Tool to setup the envtest framework to run the controllers integration tests
|
||||
# Note: With Golang 1.22 we need to use the release-0.18 branch
|
||||
ENV SETUP_ENVTEST_VERSION release-0.18
|
||||
RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@${SETUP_ENVTEST_VERSION} && \
|
||||
ENVTEST_BIN=$(setup-envtest use -p path) && \
|
||||
mkdir -p /usr/local/kubebuilder/bin && \
|
||||
cp $ENVTEST_BIN/* /usr/local/kubebuilder/bin
|
||||
|
||||
ENV GO111MODULE on
|
||||
ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS GITHUB_TOKEN
|
||||
ENV DAPPER_SOURCE /go/src/github.com/rancher/k3k/
|
||||
|
||||
@@ -18,6 +18,8 @@ replace (
|
||||
|
||||
require (
|
||||
github.com/go-logr/zapr v1.3.0
|
||||
github.com/onsi/ginkgo/v2 v2.14.0
|
||||
github.com/onsi/gomega v1.30.0
|
||||
github.com/prometheus/client_model v0.6.1
|
||||
github.com/rancher/dynamiclistener v1.27.5
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
@@ -58,6 +60,7 @@ require (
|
||||
github.com/go-openapi/jsonpointer v0.19.6 // indirect
|
||||
github.com/go-openapi/jsonreference v0.20.2 // indirect
|
||||
github.com/go-openapi/swag v0.22.3 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
@@ -65,6 +68,7 @@ require (
|
||||
github.com/google/gnostic-models v0.6.8 // indirect
|
||||
github.com/google/go-cmp v0.6.0 // indirect
|
||||
github.com/google/gofuzz v1.2.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
|
||||
github.com/google/uuid v1.3.1 // indirect
|
||||
github.com/gorilla/mux v1.8.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
@@ -110,6 +114,7 @@ require (
|
||||
golang.org/x/term v0.15.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.16.1 // indirect
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
|
||||
|
||||
@@ -25,6 +25,9 @@ github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k=
|
||||
@@ -134,6 +137,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
|
||||
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
@@ -209,6 +213,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
@@ -301,6 +306,7 @@ golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package clusterset_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
|
||||
"github.com/rancher/k3k/pkg/controller/clusterset"
|
||||
"github.com/rancher/k3k/pkg/log"
|
||||
|
||||
"go.uber.org/zap"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestController(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "ClusterSet Controller Suite")
|
||||
}
|
||||
|
||||
var (
|
||||
testEnv *envtest.Environment
|
||||
k8sClient client.Client
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
)
|
||||
|
||||
var _ = BeforeSuite(func() {
|
||||
|
||||
By("bootstrapping test environment")
|
||||
testEnv = &envtest.Environment{
|
||||
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "charts", "k3k", "crds")},
|
||||
ErrorIfCRDPathMissing: true,
|
||||
}
|
||||
cfg, err := testEnv.Start()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
scheme := buildScheme()
|
||||
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
mgr, err := ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
nopLogger := &log.Logger{SugaredLogger: zap.NewNop().Sugar()}
|
||||
|
||||
err = clusterset.Add(ctx, mgr, "", nopLogger)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
go func() {
|
||||
defer GinkgoRecover()
|
||||
err = mgr.Start(ctx)
|
||||
Expect(err).NotTo(HaveOccurred(), "failed to run manager")
|
||||
}()
|
||||
})
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
cancel()
|
||||
|
||||
By("tearing down the test environment")
|
||||
err := testEnv.Stop()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
func buildScheme() *runtime.Scheme {
|
||||
scheme := runtime.NewScheme()
|
||||
|
||||
err := corev1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = appsv1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = networkingv1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = v1alpha1.AddToScheme(scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
return scheme
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package clusterset_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
|
||||
|
||||
k3kcontroller "github.com/rancher/k3k/pkg/controller"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("ClusterSet Controller", func() {
|
||||
|
||||
Context("creating a ClusterSet", func() {
|
||||
|
||||
var (
|
||||
namespace string
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
|
||||
err := k8sClient.Create(context.Background(), createdNS)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
namespace = createdNS.Name
|
||||
})
|
||||
|
||||
When("created with a default spec", func() {
|
||||
It("should create a NetworkPolicy", func() {
|
||||
clusterSet := &v1alpha1.ClusterSet{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
GenerateName: "clusterset-",
|
||||
Namespace: namespace,
|
||||
},
|
||||
}
|
||||
|
||||
err := k8sClient.Create(ctx, clusterSet)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
// look for network policies etc
|
||||
clusterSetNetworkPolicy := &networkingv1.NetworkPolicy{}
|
||||
|
||||
Eventually(func() bool {
|
||||
deployKey := types.NamespacedName{
|
||||
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
|
||||
Namespace: namespace,
|
||||
}
|
||||
err := k8sClient.Get(ctx, deployKey, clusterSetNetworkPolicy)
|
||||
return err == nil
|
||||
}, time.Minute, time.Second).Should(BeTrue())
|
||||
|
||||
spec := clusterSetNetworkPolicy.Spec
|
||||
Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeEgress))
|
||||
Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeIngress))
|
||||
|
||||
// ingress should allow everything
|
||||
Expect(spec.Ingress).To(ConsistOf(networkingv1.NetworkPolicyIngressRule{}))
|
||||
|
||||
// egress should contains some rules
|
||||
Expect(spec.Egress).To(HaveLen(1))
|
||||
|
||||
// allow networking to all external IPs
|
||||
ipBlockRule := networkingv1.NetworkPolicyPeer{
|
||||
IPBlock: &networkingv1.IPBlock{CIDR: "0.0.0.0/0"},
|
||||
}
|
||||
|
||||
// allow networking in the same namespace
|
||||
clusterSetNamespaceRule := networkingv1.NetworkPolicyPeer{
|
||||
NamespaceSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{"kubernetes.io/metadata.name": namespace},
|
||||
},
|
||||
}
|
||||
|
||||
// allow networking to the "kube-dns" pod in the "kube-system" namespace
|
||||
kubeDNSRule := networkingv1.NetworkPolicyPeer{
|
||||
PodSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{"k8s-app": "kube-dns"},
|
||||
},
|
||||
NamespaceSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"},
|
||||
},
|
||||
}
|
||||
|
||||
Expect(spec.Egress[0].To).To(ContainElements(
|
||||
ipBlockRule, clusterSetNamespaceRule, kubeDNSRule,
|
||||
))
|
||||
})
|
||||
})
|
||||
|
||||
When("created with DisableNetworkPolicy", func() {
|
||||
It("should not create a NetworkPolicy", func() {
|
||||
clusterSet := &v1alpha1.ClusterSet{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
GenerateName: "clusterset-",
|
||||
Namespace: namespace,
|
||||
},
|
||||
Spec: v1alpha1.ClusterSetSpec{
|
||||
DisableNetworkPolicy: true,
|
||||
},
|
||||
}
|
||||
|
||||
err := k8sClient.Create(ctx, clusterSet)
|
||||
Expect(err).To(Not(HaveOccurred()))
|
||||
|
||||
// wait for a bit for the network policy, but it should not be created
|
||||
Eventually(func() bool {
|
||||
deployKey := types.NamespacedName{
|
||||
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
|
||||
Namespace: namespace,
|
||||
}
|
||||
err := k8sClient.Get(ctx, deployKey, &networkingv1.NetworkPolicy{})
|
||||
return apierrors.IsNotFound(err)
|
||||
}).
|
||||
MustPassRepeatedly(5).
|
||||
WithTimeout(time.Second * 10).
|
||||
WithPolling(time.Second).
|
||||
Should(BeTrue())
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user