diff --git a/.github/workflows/apiserver-test.yaml b/.github/workflows/apiserver-test.yaml index 3f8a63d8a..426e32b5e 100644 --- a/.github/workflows/apiserver-test.yaml +++ b/.github/workflows/apiserver-test.yaml @@ -19,9 +19,8 @@ env: # Common versions GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' - KIND_VERSION: 'v0.7.0' - KIND_IMAGE_VERSION: '[\"v1.20.7\"]' - KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]' + K3D_IMAGE_VERSION: '[\"v1.20\"]' + K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.22\"]' jobs: @@ -48,9 +47,9 @@ jobs: run: | if [[ "${{ github.ref }}" == refs/tags/v* ]]; then echo "pushing tag: ${{ github.ref_name }}" - echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}" + echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}" else - echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}" + echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}" fi apiserver-unit-tests: @@ -128,35 +127,39 @@ jobs: run: | go get -v -t -d ./... - - name: Setup Kind - uses: engineerd/setup-kind@v0.5.0 + - name: Tear down K3d if exist + run: | + k3d cluster delete || true + k3d cluster delete worker || true + + - name: Setup K3d (Hub) + uses: nolar/setup-k3d-k3s@v1.0.8 with: - version: ${{ env.KIND_VERSION }} - skipClusterCreation: true + version: ${{ matrix.k8s-version }} + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Kind Cluster (Worker) + - name: Setup K3d (Worker) + uses: nolar/setup-k3d-k3s@v1.0.8 + with: + version: ${{ matrix.k8s-version }} + github-token: ${{ secrets.GITHUB_TOKEN }} + k3d-name: worker + k3d-args: --kubeconfig-update-default=false --network=k3d-k3s-default + + - name: Kind Cluster (Worker) run: | - kind delete cluster --name worker - kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker - kubectl version - kubectl cluster-info - kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig - kind get kubeconfig --name worker > /tmp/worker.client.kubeconfig + internal_ip=$(docker network inspect k3d-k3s-default|jq ".[0].Containers"| jq -r '.[]| select(.Name=="k3d-worker-server-0")|.IPv4Address' | cut -d/ -f1) + k3d kubeconfig get worker > /tmp/worker.client.kubeconfig + cp /tmp/worker.client.kubeconfig /tmp/worker.kubeconfig + sed -i "s/0.0.0.0:[0-9]\+/$internal_ip:6443/" /tmp/worker.kubeconfig - - name: Setup Kind Cluster (Hub) - run: | - kind delete cluster - kind create cluster --image kindest/node:${{ matrix.k8s-version }} - kubectl version - kubectl cluster-info - - - name: Load Image to kind cluster - run: make kind-load + - name: Load image to k3d cluster + run: make image-load - name: Cleanup for e2e tests run: | - make e2e-cleanup make vela-cli + make e2e-cleanup make e2e-setup-core bin/vela addon enable fluxcd timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $? -eq 0 ] ; then break; else sleep 5; fi;done' diff --git a/.github/workflows/e2e-multicluster-test.yml b/.github/workflows/e2e-multicluster-test.yml index d5ef20438..974c55569 100644 --- a/.github/workflows/e2e-multicluster-test.yml +++ b/.github/workflows/e2e-multicluster-test.yml @@ -17,9 +17,8 @@ env: # Common versions GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' - KIND_VERSION: 'v0.7.0' - KIND_IMAGE_VERSION: '[\"v1.20.7\"]' - KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]' + K3D_IMAGE_VERSION: '[\"v1.20\"]' + K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.22\"]' jobs: @@ -46,9 +45,9 @@ jobs: run: | if [[ "${{ github.ref }}" == refs/tags/v* ]]; then echo "pushing tag: ${{ github.ref_name }}" - echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}" + echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}" else - echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}" + echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}" fi @@ -74,37 +73,40 @@ jobs: run: | go get -v -t -d ./... - - name: Setup Kind - uses: engineerd/setup-kind@v0.5.0 + - name: Tear down K3d if exist + run: | + k3d cluster delete || true + k3d cluster delete worker || true + + - name: Setup K3d (Hub) + uses: nolar/setup-k3d-k3s@v1.0.8 with: - version: ${{ env.KIND_VERSION }} - skipClusterCreation: true + version: ${{ matrix.k8s-version }} + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Setup Kind Cluster (Worker) + - name: Setup K3d (Worker) + uses: nolar/setup-k3d-k3s@v1.0.8 + with: + version: ${{ matrix.k8s-version }} + github-token: ${{ secrets.GITHUB_TOKEN }} + k3d-name: worker + k3d-args: --kubeconfig-update-default=false --network=k3d-k3s-default + + - name: Generating internal worker kubeconfig run: | - kind delete cluster --name worker - kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker - kubectl version - kubectl cluster-info - kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig - kind get kubeconfig --name worker > /tmp/worker.client.kubeconfig + internal_ip=$(docker network inspect k3d-k3s-default|jq ".[0].Containers"| jq -r '.[]| select(.Name=="k3d-worker-server-0")|.IPv4Address' | cut -d/ -f1) + k3d kubeconfig get worker > /tmp/worker.client.kubeconfig + cp /tmp/worker.client.kubeconfig /tmp/worker.kubeconfig + sed -i "s/0.0.0.0:[0-9]\+/$internal_ip:6443/" /tmp/worker.kubeconfig - - name: Setup Kind Cluster (Hub) - run: | - kind delete cluster - kind create cluster --image kindest/node:${{ matrix.k8s-version }} - kubectl version - kubectl cluster-info - - - name: Load Image to kind cluster (Hub) - run: make kind-load + - name: Load image to k3d cluster (hub and worker) + run: make image-load image-load-runtime-cluster - name: Cleanup for e2e tests run: | - make e2e-cleanup make vela-cli + make e2e-cleanup make e2e-setup-core-auth - make make setup-runtime-e2e-cluster - name: Run e2e multicluster tests diff --git a/.github/workflows/e2e-rollout-test.yml b/.github/workflows/e2e-rollout-test.yml index d27dc4e3f..b688fbdec 100644 --- a/.github/workflows/e2e-rollout-test.yml +++ b/.github/workflows/e2e-rollout-test.yml @@ -17,9 +17,8 @@ env: # Common versions GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' - KIND_VERSION: 'v0.7.0' - KIND_IMAGE_VERSION: '[\"v1.20.7\"]' - KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]' + K3D_IMAGE_VERSION: '[\"v1.20\"]' + K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.22\"]' jobs: @@ -46,9 +45,9 @@ jobs: run: | if [[ "${{ github.ref }}" == refs/tags/v* ]]; then echo "pushing tag: ${{ github.ref_name }}" - echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}" + echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}" else - echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}" + echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}" fi e2e-rollout-tests: @@ -72,33 +71,26 @@ jobs: run: | go get -v -t -d ./... - - name: Setup Kind - uses: engineerd/setup-kind@v0.5.0 - with: - version: ${{ env.KIND_VERSION }} - skipClusterCreation: true - - - name: Setup Kind Cluster + - name: Tear down K3d if exist run: | - kind delete cluster - kind create cluster --image kindest/node:${{ matrix.k8s-version }} - kubectl version - kubectl cluster-info + k3d cluster delete || true + k3d cluster delete worker || true - - name: Load Image to kind cluster - run: make kind-load + - name: Setup K3d + uses: nolar/setup-k3d-k3s@v1.0.8 + with: + version: ${{ matrix.k8s-version }} + github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Run Make - run: make - - - name: Run Make Manager - run: make manager + - name: Load image to k3d cluster + run: make image-load image-load-runtime-cluster - name: Prepare for e2e tests run: | + make vela-cli make e2e-cleanup - make e2e-setup - helm lint ./charts/vela-core + make e2e-setup-core + make setup-runtime-e2e-cluster helm test -n vela-system kubevela --timeout 5m - name: Run e2e tests diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 938413b27..75e945f9f 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -17,9 +17,8 @@ env: # Common versions GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' - KIND_VERSION: 'v0.7.0' - KIND_IMAGE_VERSION: '[\"v1.20.7\"]' - KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]' + K3D_IMAGE_VERSION: '[\"v1.20\"]' + K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.22\"]' jobs: @@ -46,9 +45,9 @@ jobs: run: | if [[ "${{ github.ref }}" == refs/tags/v* ]]; then echo "pushing tag: ${{ github.ref_name }}" - echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}" + echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}" else - echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}" + echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}" fi e2e-tests: @@ -72,33 +71,27 @@ jobs: run: | go get -v -t -d ./... - - name: Setup Kind - uses: engineerd/setup-kind@v0.5.0 - with: - version: ${{ env.KIND_VERSION }} - skipClusterCreation: true - - - name: Setup Kind Cluster + - name: Tear down K3d if exist run: | - kind delete cluster - kind create cluster --image kindest/node:${{ matrix.k8s-version }} - kubectl version - kubectl cluster-info + k3d cluster delete || true + k3d cluster delete worker || true - - name: Load Image to kind cluster - run: make kind-load + - name: Setup K3d + uses: nolar/setup-k3d-k3s@v1.0.8 + with: + version: ${{ matrix.k8s-version }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Load image to k3d cluster + run: make image-load - name: Run Make run: make - - name: Run Make Manager - run: make manager - - name: Prepare for e2e tests run: | make e2e-cleanup - make e2e-setup - helm lint ./charts/vela-core + make e2e-setup-core helm test -n vela-system kubevela --timeout 5m - name: Run api e2e tests diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 5315c38d4..4b339d687 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -15,7 +15,6 @@ env: # Common versions GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' - KIND_VERSION: 'v0.7.0' jobs: diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index d594bf6db..8f30bb60a 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -15,7 +15,6 @@ env: # Common versions GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' - KIND_VERSION: 'v0.7.0' jobs: @@ -61,10 +60,11 @@ jobs: run: | sudo apt-get install -y golang-ginkgo-dev - - name: Setup Kind Cluster - uses: engineerd/setup-kind@v0.5.0 + - name: Setup K3d + uses: nolar/setup-k3d-k3s@v1.0.8 with: - version: ${{ env.KIND_VERSION }} + version: v1.20 + github-token: ${{ secrets.GITHUB_TOKEN }} - name: install Kubebuilder uses: RyanSiu1995/kubebuilder-action@v1.2 diff --git a/Makefile b/Makefile index 5366c44ca..78220024f 100644 --- a/Makefile +++ b/Makefile @@ -82,17 +82,17 @@ endif -# load docker image to the kind cluster -kind-load: kind-load-runtime-cluster +# load docker image to the k3d cluster +image-load: docker build -t $(VELA_CORE_TEST_IMAGE) -f Dockerfile.e2e . - kind load docker-image $(VELA_CORE_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_CORE_TEST_IMAGE)"; exit 1; } + k3d image import $(VELA_CORE_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_CORE_TEST_IMAGE)"; exit 1; } -kind-load-runtime-cluster: +image-load-runtime-cluster: /bin/sh hack/e2e/build_runtime_rollout.sh docker build -t $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) -f runtime/rollout/e2e/Dockerfile.e2e runtime/rollout/e2e/ rm -rf runtime/rollout/e2e/tmp - kind load docker-image $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; } - kind load docker-image $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) --name=$(RUNTIME_CLUSTER_NAME) || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; } + k3d image import $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; } + k3d cluster get $(RUNTIME_CLUSTER_NAME) && k3d image import $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) --cluster=$(RUNTIME_CLUSTER_NAME) || echo "no worker cluster" # Run tests core-test: fmt vet manifests diff --git a/charts/vela-core/templates/test/test-application.yaml b/charts/vela-core/templates/test/test-application.yaml index ea98c4c03..5b3e80270 100644 --- a/charts/vela-core/templates/test/test-application.yaml +++ b/charts/vela-core/templates/test/test-application.yaml @@ -13,12 +13,6 @@ spec: properties: image: {{ .Values.imageRegistry }}{{ .Values.test.app.repository }}:{{ .Values.test.app.tag }} port: 8000 - traits: - - type: ingress - properties: - domain: testsvc.example.com - http: - "/": 8000 --- apiVersion: v1 kind: Pod @@ -52,13 +46,5 @@ spec: kubectl -n {{ include "systemDefinitionNamespace" . }} wait --for=condition=available deployments helm-test-express-server --timeout 3m echo "deployment being available" - # wait for ingress being created - while ! [ `kubectl -n {{ include "systemDefinitionNamespace" . }} get ing helm-test-express-server | grep -v NAME | wc -l` = 1 ]; do - echo "waiting for ingress being created" - sleep 1 - done - - - echo "Application and its components are created" restartPolicy: Never diff --git a/e2e/application/application_test.go b/e2e/application/application_test.go index 27a757559..c65a58432 100644 --- a/e2e/application/application_test.go +++ b/e2e/application/application_test.go @@ -294,7 +294,7 @@ var VelaQLPodListContext = func(context string, velaQL string) bool { gomega.Expect(v.Status.Phase).To(gomega.ContainSubstring("Running")) } if v.Status.NodeName != "" { - gomega.Expect(v.Status.NodeName).To(gomega.ContainSubstring("kind-control-plane")) + gomega.Expect(v.Status.NodeName).To(gomega.ContainSubstring("k3d-k3s-default-server-0")) } if v.Metadata.Namespace != "" { gomega.Expect(v.Metadata.Namespace).To(gomega.ContainSubstring("default")) diff --git a/e2e/setup/setup_suite_test.go b/e2e/setup/setup_suite_test.go deleted file mode 100644 index 855547b5a..000000000 --- a/e2e/setup/setup_suite_test.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "testing" - - "github.com/oam-dev/kubevela/e2e" - - "github.com/onsi/ginkgo" - "github.com/onsi/gomega" -) - -var _ = ginkgo.BeforeSuite(func() { - e2e.BeforeSuit() -}, 30) - -func TestApplication(t *testing.T) { - gomega.RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "Setup Suite") -} diff --git a/e2e/setup/setup_test.go b/e2e/setup/setup_test.go deleted file mode 100644 index ab2e12540..000000000 --- a/e2e/setup/setup_test.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "github.com/onsi/ginkgo" -) - -var _ = ginkgo.Describe("Setup", func() { - -}) diff --git a/hack/e2e/end_e2e.sh b/hack/e2e/end_e2e.sh index 77126a1c4..1fa22a993 100644 --- a/hack/e2e/end_e2e.sh +++ b/hack/e2e/end_e2e.sh @@ -1,20 +1,20 @@ . ./hack/e2e/end_e2e_core.sh -OAM_CONTAINER_ID=$(docker exec kind-control-plane crictl ps | grep oam-runtime | grep --regexp '^.............' -o) -OAM_DOCKER_DIR=$(docker exec kind-control-plane crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${OAM_CONTAINER_ID}") +OAM_CONTAINER_ID=$(docker exec k3d-k3s-default-server-0 crictl ps | grep oam-runtime | grep --regexp '^.............' -o) +OAM_DOCKER_DIR=$(docker exec k3d-k3s-default-server-0 crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${OAM_CONTAINER_ID}") echo "${OAM_CONTAINER_ID}" echo "${OAM_DOCKER_DIR}" -docker exec kind-control-plane crictl exec "${OAM_CONTAINER_ID}" kill -2 1 +docker exec k3d-k3s-default-server-0 crictl exec "${OAM_CONTAINER_ID}" kill -2 1 file=$OAM_DOCKER_DIR/e2e-profile.out echo "$file" n=1 while [ $n -le 60 ];do - if_exist=$(docker exec kind-control-plane sh -c "test -f $file && echo 'ok'") + if_exist=$(docker exec k3d-k3s-default-server-0 sh -c "test -f $file && echo 'ok'") echo "$if_exist" if [ -n "$if_exist" ];then - docker exec kind-control-plane cat "$file" > /tmp/oam-e2e-profile.out + docker exec k3d-k3s-default-server-0 cat "$file" > /tmp/oam-e2e-profile.out break fi echo file not generated yet diff --git a/hack/e2e/end_e2e_core.sh b/hack/e2e/end_e2e_core.sh index 3f7eed726..8918596fc 100644 --- a/hack/e2e/end_e2e_core.sh +++ b/hack/e2e/end_e2e_core.sh @@ -1,18 +1,18 @@ -CONTAINER_ID=$(docker exec kind-control-plane crictl ps | grep 'kubevela\s' | grep --regexp '^.............' -o) -DOCKER_DIR=$(docker exec kind-control-plane crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${CONTAINER_ID}") +CONTAINER_ID=$(docker exec k3d-k3s-default-server-0 crictl ps | grep 'kubevela\s' | grep --regexp '^.............' -o) +DOCKER_DIR=$(docker exec k3d-k3s-default-server-0 crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${CONTAINER_ID}") echo "${CONTAINER_ID}" echo "${DOCKER_DIR}" -docker exec kind-control-plane crictl exec "${CONTAINER_ID}" kill -2 1 +docker exec k3d-k3s-default-server-0 crictl exec "${CONTAINER_ID}" kill -2 1 file=$DOCKER_DIR/e2e-profile.out echo "$file" n=1 while [ $n -le 60 ];do - if_exist=$(docker exec kind-control-plane sh -c "test -f $file && echo 'ok'") + if_exist=$(docker exec k3d-k3s-default-server-0 sh -c "test -f $file && echo 'ok'") echo "$if_exist" if [ -n "$if_exist" ];then - docker exec kind-control-plane cat "$file" > /tmp/e2e-profile.out + docker exec k3d-k3s-default-server-0 cat "$file" > /tmp/e2e-profile.out break fi echo file not generated yet diff --git a/hack/pressure-test/tolerate-hollow-node.cue b/hack/pressure-test/tolerate-hollow-node.cue index 0d9155719..25fb8155c 100644 --- a/hack/pressure-test/tolerate-hollow-node.cue +++ b/hack/pressure-test/tolerate-hollow-node.cue @@ -9,9 +9,9 @@ template: { patch: spec: template: spec: { tolerations: [{ - key: "node.kubernetes.io/network-unavailable" + key: "node.kubernetes.io/network-unavailable" operator: "Exists" - effect: "NoSchedule" + effect: "NoSchedule" }] } parameter: {} diff --git a/makefiles/e2e.mk b/makefiles/e2e.mk index dda4ef40a..fa7fc23ca 100644 --- a/makefiles/e2e.mk +++ b/makefiles/e2e.mk @@ -5,11 +5,18 @@ e2e-setup-core-pre-hook: .PHONY: e2e-setup-core-post-hook e2e-setup-core-post-hook: kubectl wait --for=condition=Available deployment/kubevela-vela-core -n vela-system --timeout=180s - helm upgrade --install --namespace vela-system --wait oam-rollout --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) ./runtime/rollout/charts + helm install kruise https://github.com/openkruise/charts/releases/download/kruise-1.1.0/kruise-1.1.0.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true" --set daemon.socketLocation=/run/k3s/containerd/ go run ./e2e/addon/mock & sleep 15 bin/vela addon enable ./e2e/addon/mock/testdata/fluxcd bin/vela addon enable ./e2e/addon/mock/testdata/rollout + bin/vela addon enable ./e2e/addon/mock/testdata/terraform + bin/vela addon enable ./e2e/addon/mock/testdata/terraform-alibaba ALICLOUD_ACCESS_KEY=xxx ALICLOUD_SECRET_KEY=yyy ALICLOUD_REGION=cn-beijing + + timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $$? -eq 0 ] ; then break; else sleep 5; fi;done' + kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s + kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s + kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s .PHONY: e2e-setup-core-wo-auth e2e-setup-core-wo-auth: @@ -27,28 +34,26 @@ e2e-setup-core-auth: e2e-setup-core-pre-hook e2e-setup-core-w-auth e2e-setup-cor .PHONY: setup-runtime-e2e-cluster setup-runtime-e2e-cluster: - helm upgrade --install --create-namespace --namespace vela-system --kubeconfig=$(RUNTIME_CLUSTER_CONFIG) --set image.pullPolicy=IfNotPresent --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) --wait vela-rollout ./runtime/rollout/charts + helm upgrade --install \ + --namespace vela-system \ + --wait oam-rollout \ + --set image.repository=vela-runtime-rollout-test \ + --set image.tag=$(GIT_COMMIT) \ + ./runtime/rollout/charts -.PHONY: e2e-setup -e2e-setup: - helm install kruise https://github.com/openkruise/charts/releases/download/kruise-1.1.0/kruise-1.1.0.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true" - sh ./hack/e2e/modify_charts.sh - helm upgrade --install --create-namespace --namespace vela-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set applicationRevisionLimit=5 --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --wait kubevela ./charts/vela-core - helm upgrade --install --namespace vela-system --wait oam-rollout --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) ./runtime/rollout/charts + k3d cluster get $(RUNTIME_CLUSTER_NAME) && \ + helm upgrade --install \ + --create-namespace \ + --namespace vela-system \ + --kubeconfig=$(RUNTIME_CLUSTER_CONFIG) \ + --set image.pullPolicy=IfNotPresent \ + --set image.repository=vela-runtime-rollout-test \ + --set image.tag=$(GIT_COMMIT) \ + --wait vela-rollout \ + ./runtime/rollout/charts || \ + echo "no worker cluster" \ - go run ./e2e/addon/mock & - sleep 15 - bin/vela addon enable ./e2e/addon/mock/testdata/fluxcd - bin/vela addon enable ./e2e/addon/mock/testdata/terraform - bin/vela addon enable ./e2e/addon/mock/testdata/terraform-alibaba ALICLOUD_ACCESS_KEY=xxx ALICLOUD_SECRET_KEY=yyy ALICLOUD_REGION=cn-beijing - bin/vela addon enable ./e2e/addon/mock/testdata/rollout - ginkgo version - ginkgo -v -r e2e/setup - timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $$? -eq 0 ] ; then break; else sleep 5; fi;done' - kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s - kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s - kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s .PHONY: e2e-api-test e2e-api-test: diff --git a/test/e2e-multicluster-test/suite_test.go b/test/e2e-multicluster-test/suite_test.go index 2c5732041..2d3d5427b 100644 --- a/test/e2e-multicluster-test/suite_test.go +++ b/test/e2e-multicluster-test/suite_test.go @@ -23,11 +23,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/strings/slices" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/multicluster" + oamutil "github.com/oam-dev/kubevela/pkg/oam/util" "github.com/oam-dev/kubevela/pkg/utils/common" "github.com/oam-dev/kubevela/pkg/utils/util" "github.com/oam-dev/kubevela/references/cli" @@ -73,12 +77,25 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { + holdAddons := []string{"addon-terraform", "addon-fluxcd"} Eventually(func(g Gomega) { apps := &v1beta1.ApplicationList{} g.Expect(k8sClient.List(context.Background(), apps)).Should(Succeed()) for _, app := range apps.Items { + if slices.Contains(holdAddons, app.Name) { + continue + } g.Expect(k8sClient.Delete(context.Background(), app.DeepCopy())).Should(Succeed()) } + }, 3*time.Minute).Should(Succeed()) + Eventually(func(g Gomega) { + // Delete terraform and fluxcd in order + app := &v1beta1.Application{} + apps := &v1beta1.ApplicationList{} + for _, addon := range holdAddons { + g.Expect(k8sClient.Delete(context.Background(), &v1beta1.Application{ObjectMeta: v1.ObjectMeta{Name: addon, Namespace: types.DefaultKubeVelaNS}})).Should(SatisfyAny(Succeed(), oamutil.NotFoundMatcher{})) + g.Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: addon, Namespace: types.DefaultKubeVelaNS}, app)).Should(SatisfyAny(Succeed(), oamutil.NotFoundMatcher{})) + } err := k8sClient.List(context.Background(), apps) g.Expect(err, nil) g.Expect(len(apps.Items)).Should(Equal(0)) diff --git a/test/e2e-multicluster-test/testdata/app/app-bad-resource.yaml b/test/e2e-multicluster-test/testdata/app/app-bad-resource.yaml index f630ffbd9..6f9be1360 100644 --- a/test/e2e-multicluster-test/testdata/app/app-bad-resource.yaml +++ b/test/e2e-multicluster-test/testdata/app/app-bad-resource.yaml @@ -8,7 +8,7 @@ spec: - name: bad-resource properties: objects: - - apiVersion: apiregistration.k8s.io/v1beta1 + - apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: name: test-bad-resource diff --git a/test/e2e-test/health_scope_test.go b/test/e2e-test/health_scope_test.go index f21962cc4..826b8c835 100644 --- a/test/e2e-test/health_scope_test.go +++ b/test/e2e-test/health_scope_test.go @@ -18,6 +18,7 @@ package controllers_test import ( "context" + "encoding/json" "fmt" "strings" "time" @@ -27,6 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -80,6 +82,7 @@ var _ = Describe("HealthScope", func() { var healthyAppName, unhealthyAppName string Expect(utilcommon.ReadYamlToObject("testdata/app/app_healthscope.yaml", &newApp)).Should(BeNil()) newApp.Namespace = namespace + convertToLegacyIngressTrait(&newApp) Eventually(func() error { return k8sClient.Create(ctx, newApp.DeepCopy()) }, 10*time.Second, 500*time.Millisecond).Should(Succeed()) @@ -101,6 +104,7 @@ var _ = Describe("HealthScope", func() { newApp = v1beta1.Application{} Expect(utilcommon.ReadYamlToObject("testdata/app/app_healthscope_unhealthy.yaml", &newApp)).Should(BeNil()) newApp.Namespace = namespace + convertToLegacyIngressTrait(&newApp) Eventually(func() error { return k8sClient.Create(ctx, newApp.DeepCopy()) }, 10*time.Second, 500*time.Millisecond).Should(Succeed()) @@ -154,9 +158,11 @@ var _ = Describe("HealthScope", func() { return fmt.Errorf("expect healthy comp, but %v is unhealthy, msg: %q", compSts1.Name, compSts1.Message) } if len(compSts1.Traits) != 1 { - return fmt.Errorf("expect 2 traits statuses, but got %d", len(compSts1.Traits)) + return fmt.Errorf("expect 1 trait statuses, but got %d", len(compSts1.Traits)) + } + if !strings.Contains(compSts1.Traits[0].Message, "visit the cluster or load balancer in front of the cluster") { + return fmt.Errorf("trait message isn't right, now is %s", compSts1.Traits[0].Message) } - Expect(compSts1.Traits[0].Message).Should(ContainSubstring("No loadBalancer found")) return nil }, time.Second*30, time.Millisecond*500).Should(Succeed()) @@ -205,3 +211,34 @@ var _ = Describe("HealthScope", func() { }, time.Second*30, time.Millisecond*500).Should(Succeed()) }) }) + +// convertToLegacyIngressTrait convert app's gateway trait to ingress +func convertToLegacyIngressTrait(app *v1beta1.Application) { + if noNetworkingV1 { + for i := range app.Spec.Components { + for j := range app.Spec.Components[i].Traits { + if app.Spec.Components[i].Traits[j].Type == "gateway" { + app.Spec.Components[i].Traits[j].Type = "ingress" + } + props := app.Spec.Components[i].Traits[j].Properties + propMap, err := util.RawExtension2Map(props) + if err != nil { + return + } + newPropMap := map[string]interface{}{} + for k := range propMap { + if k != "class" { + newPropMap[k] = propMap[k] + } + } + ext, err := json.Marshal(newPropMap) + if err != nil { + return + } + app.Spec.Components[i].Traits[j].Properties = &runtime.RawExtension{ + Raw: ext, + } + } + } + } +} diff --git a/test/e2e-test/suite_test.go b/test/e2e-test/suite_test.go index 96d85e2e8..80f0f25ca 100644 --- a/test/e2e-test/suite_test.go +++ b/test/e2e-test/suite_test.go @@ -31,13 +31,17 @@ import ( . "github.com/onsi/gomega" kruise "github.com/openkruise/kruise-api/apps/v1alpha1" + "github.com/pkg/errors" + networkv1 "k8s.io/api/networking/v1" rbac "k8s.io/api/rbac/v1" crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + k8sutils "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" @@ -58,6 +62,10 @@ var manualscalertrait v1alpha2.TraitDefinition var roleName = "oam-example-com" var roleBindingName = "oam-role-binding" +var ( + noNetworkingV1 bool +) + // A DefinitionExtension is an Object type for xxxDefinitin.spec.extension type DefinitionExtension struct { Alias string `json:"alias,omitempty"` @@ -104,6 +112,8 @@ var _ = BeforeSuite(func(done Done) { } By("Finished setting up test environment") + detectAPIVersion() + // Create manual scaler trait definition manualscalertrait = v1alpha2.TraitDefinition{ ObjectMeta: metav1.ObjectMeta{ @@ -219,3 +229,20 @@ func RequestReconcileNow(ctx context.Context, o client.Object) { func randomNamespaceName(basic string) string { return fmt.Sprintf("%s-%s", basic, strconv.FormatInt(rand.Int63(), 16)) } + +// detectAPIVersion helps detect legacy GVK +func detectAPIVersion() { + err := k8sClient.Create(context.Background(), &networkv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + }, + Spec: networkv1.IngressSpec{ + IngressClassName: k8sutils.StringPtr("nginx"), + Rules: []networkv1.IngressRule{}, + }, + }) + var noKindMatchErr = &meta.NoKindMatchError{} + if err != nil && errors.As(err, &noKindMatchErr) { + noNetworkingV1 = true + } +} diff --git a/test/e2e-test/testdata/app/app_healthscope.yaml b/test/e2e-test/testdata/app/app_healthscope.yaml index 8b1410eb1..971f94c4a 100644 --- a/test/e2e-test/testdata/app/app_healthscope.yaml +++ b/test/e2e-test/testdata/app/app_healthscope.yaml @@ -13,8 +13,9 @@ spec: image: oamdev/testapp:v1 port: 8080 traits: - - type: ingress + - type: gateway properties: + class: traefik domain: test.my.domain http: "/": 8080 @@ -27,8 +28,9 @@ spec: image: oamdev/testapp:v1 port: 8080 traits: - - type: ingress + - type: gateway properties: + class: traefik domain: test.my.domain http: "/": 8080 diff --git a/test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml b/test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml index f2ab3efd1..5a6675088 100644 --- a/test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml +++ b/test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml @@ -21,8 +21,9 @@ spec: image: oamdev/testapp:v1 port: 8080 traits: - - type: ingress + - type: gateway properties: + class: traefik domain: test.my.domain http: "/": 8080