github/kubevela
1
0
Fork 0
mirror of https://github.com/kubevela/kubevela.git synced 2026-08-27 16:17:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Chore: Migrate to k3d in CI & trim redundant steps (#4652) (#4676)

Browse Source 
Fix: flaky CI test (#4669)
Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>

Signed-off-by: Qiaozp <qiaozhongpei.qzp@alibaba-inc.com>
This commit is contained in:
qiaozp
2022-09-01 17:22:02 +08:00
committed by GitHub
parent e460638799
commit 44142c4c70
21 changed files with 229 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/apiserver-test.yaml
+29 -26
View File
@@ -19,9 +19,8 @@ env:
# Common versions
GO_VERSION: '1.17'
GOLANGCI_VERSION: 'v1.38'
KIND_VERSION: 'v0.7.0'
KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'
K3D_IMAGE_VERSION: '[\"v1.20\"]'
K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.22\"]'
jobs:
@@ -48,9 +47,9 @@ jobs:
run: |
if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
echo "pushing tag: ${{ github.ref_name }}"
echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}"
else
echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}"
fi
apiserver-unit-tests:
@@ -128,35 +127,39 @@ jobs:
run: |
go get -v -t -d ./...
- name: Setup Kind
uses: engineerd/setup-kind@v0.5.0
- name: Tear down K3d if exist
run: |
k3d cluster delete || true
k3d cluster delete worker || true
- name: Setup K3d (Hub)
uses: nolar/setup-k3d-k3s@v1.0.8
with:
version: ${{ env.KIND_VERSION }}
skipClusterCreation: true
version: ${{ matrix.k8s-version }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Kind Cluster (Worker)
- name: Setup K3d (Worker)
uses: nolar/setup-k3d-k3s@v1.0.8
with:
version: ${{ matrix.k8s-version }}
github-token: ${{ secrets.GITHUB_TOKEN }}
k3d-name: worker
k3d-args: --kubeconfig-update-default=false --network=k3d-k3s-default
- name: Kind Cluster (Worker)
run: |
kind delete cluster --name worker
kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker
kubectl version
kubectl cluster-info
kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig
kind get kubeconfig --name worker > /tmp/worker.client.kubeconfig
internal_ip=$(docker network inspect k3d-k3s-default|jq ".[0].Containers"| jq -r '.[]| select(.Name=="k3d-worker-server-0")|.IPv4Address' | cut -d/ -f1)
k3d kubeconfig get worker > /tmp/worker.client.kubeconfig
cp /tmp/worker.client.kubeconfig /tmp/worker.kubeconfig
sed -i "s/0.0.0.0:[0-9]\+/$internal_ip:6443/" /tmp/worker.kubeconfig
- name: Setup Kind Cluster (Hub)
run: |
kind delete cluster
kind create cluster --image kindest/node:${{ matrix.k8s-version }}
kubectl version
kubectl cluster-info
- name: Load Image to kind cluster
run: make kind-load
- name: Load image to k3d cluster
run: make image-load
- name: Cleanup for e2e tests
run: |
make e2e-cleanup
make vela-cli
make e2e-cleanup
make e2e-setup-core
bin/vela addon enable fluxcd
timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $? -eq 0 ] ; then break; else sleep 5; fi;done'
.github/workflows/e2e-multicluster-test.yml
+29 -27
View File
@@ -17,9 +17,8 @@ env:
# Common versions
GO_VERSION: '1.17'
GOLANGCI_VERSION: 'v1.38'
KIND_VERSION: 'v0.7.0'
KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'
K3D_IMAGE_VERSION: '[\"v1.20\"]'
K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.22\"]'
jobs:
@@ -46,9 +45,9 @@ jobs:
run: |
if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
echo "pushing tag: ${{ github.ref_name }}"
echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}"
else
echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}"
fi
@@ -74,37 +73,40 @@ jobs:
run: |
go get -v -t -d ./...
- name: Setup Kind
uses: engineerd/setup-kind@v0.5.0
- name: Tear down K3d if exist
run: |
k3d cluster delete || true
k3d cluster delete worker || true
- name: Setup K3d (Hub)
uses: nolar/setup-k3d-k3s@v1.0.8
with:
version: ${{ env.KIND_VERSION }}
skipClusterCreation: true
version: ${{ matrix.k8s-version }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Kind Cluster (Worker)
- name: Setup K3d (Worker)
uses: nolar/setup-k3d-k3s@v1.0.8
with:
version: ${{ matrix.k8s-version }}
github-token: ${{ secrets.GITHUB_TOKEN }}
k3d-name: worker
k3d-args: --kubeconfig-update-default=false --network=k3d-k3s-default
- name: Generating internal worker kubeconfig
run: |
kind delete cluster --name worker
kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker
kubectl version
kubectl cluster-info
kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig
kind get kubeconfig --name worker > /tmp/worker.client.kubeconfig
internal_ip=$(docker network inspect k3d-k3s-default|jq ".[0].Containers"| jq -r '.[]| select(.Name=="k3d-worker-server-0")|.IPv4Address' | cut -d/ -f1)
k3d kubeconfig get worker > /tmp/worker.client.kubeconfig
cp /tmp/worker.client.kubeconfig /tmp/worker.kubeconfig
sed -i "s/0.0.0.0:[0-9]\+/$internal_ip:6443/" /tmp/worker.kubeconfig
- name: Setup Kind Cluster (Hub)
run: |
kind delete cluster
kind create cluster --image kindest/node:${{ matrix.k8s-version }}
kubectl version
kubectl cluster-info
- name: Load Image to kind cluster (Hub)
run: make kind-load
- name: Load image to k3d cluster (hub and worker)
run: make image-load image-load-runtime-cluster
- name: Cleanup for e2e tests
run: |
make e2e-cleanup
make vela-cli
make e2e-cleanup
make e2e-setup-core-auth
make
make setup-runtime-e2e-cluster
- name: Run e2e multicluster tests
.github/workflows/e2e-rollout-test.yml
+17 -25
View File
@@ -17,9 +17,8 @@ env:
# Common versions
GO_VERSION: '1.17'
GOLANGCI_VERSION: 'v1.38'
KIND_VERSION: 'v0.7.0'
KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'
K3D_IMAGE_VERSION: '[\"v1.20\"]'
K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.22\"]'
jobs:
@@ -46,9 +45,9 @@ jobs:
run: |
if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
echo "pushing tag: ${{ github.ref_name }}"
echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}"
else
echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}"
fi
e2e-rollout-tests:
@@ -72,33 +71,26 @@ jobs:
run: |
go get -v -t -d ./...
- name: Setup Kind
uses: engineerd/setup-kind@v0.5.0
with:
version: ${{ env.KIND_VERSION }}
skipClusterCreation: true
- name: Setup Kind Cluster
- name: Tear down K3d if exist
run: |
kind delete cluster
kind create cluster --image kindest/node:${{ matrix.k8s-version }}
kubectl version
kubectl cluster-info
k3d cluster delete || true
k3d cluster delete worker || true
- name: Load Image to kind cluster
run: make kind-load
- name: Setup K3d
uses: nolar/setup-k3d-k3s@v1.0.8
with:
version: ${{ matrix.k8s-version }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Run Make
run: make
- name: Run Make Manager
run: make manager
- name: Load image to k3d cluster
run: make image-load image-load-runtime-cluster
- name: Prepare for e2e tests
run: |
make vela-cli
make e2e-cleanup
make e2e-setup
helm lint ./charts/vela-core
make e2e-setup-core
make setup-runtime-e2e-cluster
helm test -n vela-system kubevela --timeout 5m
- name: Run e2e tests
.github/workflows/e2e-test.yml
+16 -23
View File
@@ -17,9 +17,8 @@ env:
# Common versions
GO_VERSION: '1.17'
GOLANGCI_VERSION: 'v1.38'
KIND_VERSION: 'v0.7.0'
KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'
K3D_IMAGE_VERSION: '[\"v1.20\"]'
K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.22\"]'
jobs:
@@ -46,9 +45,9 @@ jobs:
run: |
if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
echo "pushing tag: ${{ github.ref_name }}"
echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}"
else
echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}"
fi
e2e-tests:
@@ -72,33 +71,27 @@ jobs:
run: |
go get -v -t -d ./...
- name: Setup Kind
uses: engineerd/setup-kind@v0.5.0
with:
version: ${{ env.KIND_VERSION }}
skipClusterCreation: true
- name: Setup Kind Cluster
- name: Tear down K3d if exist
run: |
kind delete cluster
kind create cluster --image kindest/node:${{ matrix.k8s-version }}
kubectl version
kubectl cluster-info
k3d cluster delete || true
k3d cluster delete worker || true
- name: Load Image to kind cluster
run: make kind-load
- name: Setup K3d
uses: nolar/setup-k3d-k3s@v1.0.8
with:
version: ${{ matrix.k8s-version }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Load image to k3d cluster
run: make image-load
- name: Run Make
run: make
- name: Run Make Manager
run: make manager
- name: Prepare for e2e tests
run: |
make e2e-cleanup
make e2e-setup
helm lint ./charts/vela-core
make e2e-setup-core
helm test -n vela-system kubevela --timeout 5m
- name: Run api e2e tests
.github/workflows/go.yml
-1
View File
@@ -15,7 +15,6 @@ env:
# Common versions
GO_VERSION: '1.17'
GOLANGCI_VERSION: 'v1.38'
KIND_VERSION: 'v0.7.0'
jobs:
.github/workflows/unit-test.yml
+4 -4
View File
@@ -15,7 +15,6 @@ env:
# Common versions
GO_VERSION: '1.17'
GOLANGCI_VERSION: 'v1.38'
KIND_VERSION: 'v0.7.0'
jobs:
@@ -61,10 +60,11 @@ jobs:
run: |
sudo apt-get install -y golang-ginkgo-dev
- name: Setup Kind Cluster
uses: engineerd/setup-kind@v0.5.0
- name: Setup K3d
uses: nolar/setup-k3d-k3s@v1.0.8
with:
version: ${{ env.KIND_VERSION }}
version: v1.20
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: install Kubebuilder
uses: RyanSiu1995/kubebuilder-action@v1.2
Makefile
+6 -6
View File
@@ -82,17 +82,17 @@ endif
# load docker image to the kind cluster
kind-load: kind-load-runtime-cluster
# load docker image to the k3d cluster
image-load:
docker build -t $(VELA_CORE_TEST_IMAGE) -f Dockerfile.e2e .
kind load docker-image $(VELA_CORE_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_CORE_TEST_IMAGE)"; exit 1; }
k3d image import $(VELA_CORE_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_CORE_TEST_IMAGE)"; exit 1; }
kind-load-runtime-cluster:
image-load-runtime-cluster:
/bin/sh hack/e2e/build_runtime_rollout.sh
docker build -t $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) -f runtime/rollout/e2e/Dockerfile.e2e runtime/rollout/e2e/
rm -rf runtime/rollout/e2e/tmp
kind load docker-image $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; }
kind load docker-image $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) --name=$(RUNTIME_CLUSTER_NAME) || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; }
k3d image import $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; }
k3d cluster get $(RUNTIME_CLUSTER_NAME) && k3d image import $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) --cluster=$(RUNTIME_CLUSTER_NAME) || echo "no worker cluster"
# Run tests
core-test: fmt vet manifests
charts/vela-core/templates/test/test-application.yaml
-14
View File
@@ -13,12 +13,6 @@ spec:
properties:
image: {{ .Values.imageRegistry }}{{ .Values.test.app.repository }}:{{ .Values.test.app.tag }}
port: 8000
traits:
- type: ingress
properties:
domain: testsvc.example.com
http:
"/": 8000
---
apiVersion: v1
kind: Pod
@@ -52,13 +46,5 @@ spec:
kubectl -n {{ include "systemDefinitionNamespace" . }} wait --for=condition=available deployments helm-test-express-server --timeout 3m
echo "deployment being available"
# wait for ingress being created
while ! [ `kubectl -n {{ include "systemDefinitionNamespace" . }} get ing helm-test-express-server | grep -v NAME | wc -l` = 1 ]; do
echo "waiting for ingress being created"
sleep 1
done
echo "Application and its components are created"
restartPolicy: Never
e2e/application/application_test.go
+1 -1
View File
@@ -294,7 +294,7 @@ var VelaQLPodListContext = func(context string, velaQL string) bool {
gomega.Expect(v.Status.Phase).To(gomega.ContainSubstring("Running"))
}
if v.Status.NodeName != "" {
gomega.Expect(v.Status.NodeName).To(gomega.ContainSubstring("kind-control-plane"))
gomega.Expect(v.Status.NodeName).To(gomega.ContainSubstring("k3d-k3s-default-server-0"))
}
if v.Metadata.Namespace != "" {
gomega.Expect(v.Metadata.Namespace).To(gomega.ContainSubstring("default"))
e2e/setup/setup_suite_test.go
-35
View File
@@ -1,35 +0,0 @@
/*
Copyright 2021 The KubeVela Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"testing"
"github.com/oam-dev/kubevela/e2e"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
)
var _ = ginkgo.BeforeSuite(func() {
e2e.BeforeSuit()
}, 30)
func TestApplication(t *testing.T) {
gomega.RegisterFailHandler(ginkgo.Fail)
ginkgo.RunSpecs(t, "Setup Suite")
}
e2e/setup/setup_test.go
-25
View File
@@ -1,25 +0,0 @@
/*
Copyright 2021 The KubeVela Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"github.com/onsi/ginkgo"
)
var _ = ginkgo.Describe("Setup", func() {
})
hack/e2e/end_e2e.sh
+5 -5
View File
@@ -1,20 +1,20 @@
. ./hack/e2e/end_e2e_core.sh
OAM_CONTAINER_ID=$(docker exec kind-control-plane crictl ps | grep oam-runtime | grep --regexp '^.............' -o)
OAM_DOCKER_DIR=$(docker exec kind-control-plane crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${OAM_CONTAINER_ID}")
OAM_CONTAINER_ID=$(docker exec k3d-k3s-default-server-0 crictl ps | grep oam-runtime | grep --regexp '^.............' -o)
OAM_DOCKER_DIR=$(docker exec k3d-k3s-default-server-0 crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${OAM_CONTAINER_ID}")
echo "${OAM_CONTAINER_ID}"
echo "${OAM_DOCKER_DIR}"
docker exec kind-control-plane crictl exec "${OAM_CONTAINER_ID}" kill -2 1
docker exec k3d-k3s-default-server-0 crictl exec "${OAM_CONTAINER_ID}" kill -2 1
file=$OAM_DOCKER_DIR/e2e-profile.out
echo "$file"
n=1
while [ $n -le 60 ];do
if_exist=$(docker exec kind-control-plane sh -c "test -f $file && echo 'ok'")
if_exist=$(docker exec k3d-k3s-default-server-0 sh -c "test -f $file && echo 'ok'")
echo "$if_exist"
if [ -n "$if_exist" ];then
docker exec kind-control-plane cat "$file" > /tmp/oam-e2e-profile.out
docker exec k3d-k3s-default-server-0 cat "$file" > /tmp/oam-e2e-profile.out
break
fi
echo file not generated yet
hack/e2e/end_e2e_core.sh
+5 -5
View File
@@ -1,18 +1,18 @@
CONTAINER_ID=$(docker exec kind-control-plane crictl ps | grep 'kubevela\s' | grep --regexp '^.............' -o)
DOCKER_DIR=$(docker exec kind-control-plane crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${CONTAINER_ID}")
CONTAINER_ID=$(docker exec k3d-k3s-default-server-0 crictl ps | grep 'kubevela\s' | grep --regexp '^.............' -o)
DOCKER_DIR=$(docker exec k3d-k3s-default-server-0 crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${CONTAINER_ID}")
echo "${CONTAINER_ID}"
echo "${DOCKER_DIR}"
docker exec kind-control-plane crictl exec "${CONTAINER_ID}" kill -2 1
docker exec k3d-k3s-default-server-0 crictl exec "${CONTAINER_ID}" kill -2 1
file=$DOCKER_DIR/e2e-profile.out
echo "$file"
n=1
while [ $n -le 60 ];do
if_exist=$(docker exec kind-control-plane sh -c "test -f $file && echo 'ok'")
if_exist=$(docker exec k3d-k3s-default-server-0 sh -c "test -f $file && echo 'ok'")
echo "$if_exist"
if [ -n "$if_exist" ];then
docker exec kind-control-plane cat "$file" > /tmp/e2e-profile.out
docker exec k3d-k3s-default-server-0 cat "$file" > /tmp/e2e-profile.out
break
fi
echo file not generated yet
hack/pressure-test/tolerate-hollow-node.cue
+2 -2
View File
@@ -9,9 +9,9 @@
template: {
patch: spec: template: spec: {
tolerations: [{
key: "node.kubernetes.io/network-unavailable"
key: "node.kubernetes.io/network-unavailable"
operator: "Exists"
effect: "NoSchedule"
effect: "NoSchedule"
}]
}
parameter: {}
makefiles/e2e.mk
+25 -20
View File
@@ -5,11 +5,18 @@ e2e-setup-core-pre-hook:
.PHONY: e2e-setup-core-post-hook
e2e-setup-core-post-hook:
kubectl wait --for=condition=Available deployment/kubevela-vela-core -n vela-system --timeout=180s
helm upgrade --install --namespace vela-system --wait oam-rollout --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) ./runtime/rollout/charts
helm install kruise https://github.com/openkruise/charts/releases/download/kruise-1.1.0/kruise-1.1.0.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true" --set daemon.socketLocation=/run/k3s/containerd/
go run ./e2e/addon/mock &
sleep 15
bin/vela addon enable ./e2e/addon/mock/testdata/fluxcd
bin/vela addon enable ./e2e/addon/mock/testdata/rollout
bin/vela addon enable ./e2e/addon/mock/testdata/terraform
bin/vela addon enable ./e2e/addon/mock/testdata/terraform-alibaba ALICLOUD_ACCESS_KEY=xxx ALICLOUD_SECRET_KEY=yyy ALICLOUD_REGION=cn-beijing
timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $$? -eq 0 ] ; then break; else sleep 5; fi;done'
kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s
kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s
kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s
.PHONY: e2e-setup-core-wo-auth
e2e-setup-core-wo-auth:
@@ -27,28 +34,26 @@ e2e-setup-core-auth: e2e-setup-core-pre-hook e2e-setup-core-w-auth e2e-setup-cor
.PHONY: setup-runtime-e2e-cluster
setup-runtime-e2e-cluster:
helm upgrade --install --create-namespace --namespace vela-system --kubeconfig=$(RUNTIME_CLUSTER_CONFIG) --set image.pullPolicy=IfNotPresent --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) --wait vela-rollout ./runtime/rollout/charts
helm upgrade --install \
--namespace vela-system \
--wait oam-rollout \
--set image.repository=vela-runtime-rollout-test \
--set image.tag=$(GIT_COMMIT) \
./runtime/rollout/charts
.PHONY: e2e-setup
e2e-setup:
helm install kruise https://github.com/openkruise/charts/releases/download/kruise-1.1.0/kruise-1.1.0.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true"
sh ./hack/e2e/modify_charts.sh
helm upgrade --install --create-namespace --namespace vela-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set applicationRevisionLimit=5 --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --wait kubevela ./charts/vela-core
helm upgrade --install --namespace vela-system --wait oam-rollout --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) ./runtime/rollout/charts
k3d cluster get $(RUNTIME_CLUSTER_NAME) && \
helm upgrade --install \
--create-namespace \
--namespace vela-system \
--kubeconfig=$(RUNTIME_CLUSTER_CONFIG) \
--set image.pullPolicy=IfNotPresent \
--set image.repository=vela-runtime-rollout-test \
--set image.tag=$(GIT_COMMIT) \
--wait vela-rollout \
./runtime/rollout/charts || \
echo "no worker cluster" \
go run ./e2e/addon/mock &
sleep 15
bin/vela addon enable ./e2e/addon/mock/testdata/fluxcd
bin/vela addon enable ./e2e/addon/mock/testdata/terraform
bin/vela addon enable ./e2e/addon/mock/testdata/terraform-alibaba ALICLOUD_ACCESS_KEY=xxx ALICLOUD_SECRET_KEY=yyy ALICLOUD_REGION=cn-beijing
bin/vela addon enable ./e2e/addon/mock/testdata/rollout
ginkgo version
ginkgo -v -r e2e/setup
timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $$? -eq 0 ] ; then break; else sleep 5; fi;done'
kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s
kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s
kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s
.PHONY: e2e-api-test
e2e-api-test:
test/e2e-multicluster-test/suite_test.go
+17
View File
@@ -23,11 +23,15 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/strings/slices"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
"github.com/oam-dev/kubevela/apis/types"
"github.com/oam-dev/kubevela/pkg/multicluster"
oamutil "github.com/oam-dev/kubevela/pkg/oam/util"
"github.com/oam-dev/kubevela/pkg/utils/common"
"github.com/oam-dev/kubevela/pkg/utils/util"
"github.com/oam-dev/kubevela/references/cli"
@@ -73,12 +77,25 @@ var _ = BeforeSuite(func() {
})
var _ = AfterSuite(func() {
holdAddons := []string{"addon-terraform", "addon-fluxcd"}
Eventually(func(g Gomega) {
apps := &v1beta1.ApplicationList{}
g.Expect(k8sClient.List(context.Background(), apps)).Should(Succeed())
for _, app := range apps.Items {
if slices.Contains(holdAddons, app.Name) {
continue
}
g.Expect(k8sClient.Delete(context.Background(), app.DeepCopy())).Should(Succeed())
}
}, 3*time.Minute).Should(Succeed())
Eventually(func(g Gomega) {
// Delete terraform and fluxcd in order
app := &v1beta1.Application{}
apps := &v1beta1.ApplicationList{}
for _, addon := range holdAddons {
g.Expect(k8sClient.Delete(context.Background(), &v1beta1.Application{ObjectMeta: v1.ObjectMeta{Name: addon, Namespace: types.DefaultKubeVelaNS}})).Should(SatisfyAny(Succeed(), oamutil.NotFoundMatcher{}))
g.Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: addon, Namespace: types.DefaultKubeVelaNS}, app)).Should(SatisfyAny(Succeed(), oamutil.NotFoundMatcher{}))
}
err := k8sClient.List(context.Background(), apps)
g.Expect(err, nil)
g.Expect(len(apps.Items)).Should(Equal(0))
test/e2e-multicluster-test/testdata/app/app-bad-resource.yaml
Vendored
+1 -1
View File
@@ -8,7 +8,7 @@ spec:
- name: bad-resource
properties:
objects:
- apiVersion: apiregistration.k8s.io/v1beta1
- apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
name: test-bad-resource
test/e2e-test/health_scope_test.go
+39 -2
View File
@@ -18,6 +18,7 @@ package controllers_test
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
@@ -27,6 +28,7 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -80,6 +82,7 @@ var _ = Describe("HealthScope", func() {
var healthyAppName, unhealthyAppName string
Expect(utilcommon.ReadYamlToObject("testdata/app/app_healthscope.yaml", &newApp)).Should(BeNil())
newApp.Namespace = namespace
convertToLegacyIngressTrait(&newApp)
Eventually(func() error {
return k8sClient.Create(ctx, newApp.DeepCopy())
}, 10*time.Second, 500*time.Millisecond).Should(Succeed())
@@ -101,6 +104,7 @@ var _ = Describe("HealthScope", func() {
newApp = v1beta1.Application{}
Expect(utilcommon.ReadYamlToObject("testdata/app/app_healthscope_unhealthy.yaml", &newApp)).Should(BeNil())
newApp.Namespace = namespace
convertToLegacyIngressTrait(&newApp)
Eventually(func() error {
return k8sClient.Create(ctx, newApp.DeepCopy())
}, 10*time.Second, 500*time.Millisecond).Should(Succeed())
@@ -154,9 +158,11 @@ var _ = Describe("HealthScope", func() {
return fmt.Errorf("expect healthy comp, but %v is unhealthy, msg: %q", compSts1.Name, compSts1.Message)
}
if len(compSts1.Traits) != 1 {
return fmt.Errorf("expect 2 traits statuses, but got %d", len(compSts1.Traits))
return fmt.Errorf("expect 1 trait statuses, but got %d", len(compSts1.Traits))
}
if !strings.Contains(compSts1.Traits[0].Message, "visit the cluster or load balancer in front of the cluster") {
return fmt.Errorf("trait message isn't right, now is %s", compSts1.Traits[0].Message)
}
Expect(compSts1.Traits[0].Message).Should(ContainSubstring("No loadBalancer found"))
return nil
}, time.Second*30, time.Millisecond*500).Should(Succeed())
@@ -205,3 +211,34 @@ var _ = Describe("HealthScope", func() {
}, time.Second*30, time.Millisecond*500).Should(Succeed())
})
})
// convertToLegacyIngressTrait convert app's gateway trait to ingress
func convertToLegacyIngressTrait(app *v1beta1.Application) {
if noNetworkingV1 {
for i := range app.Spec.Components {
for j := range app.Spec.Components[i].Traits {
if app.Spec.Components[i].Traits[j].Type == "gateway" {
app.Spec.Components[i].Traits[j].Type = "ingress"
}
props := app.Spec.Components[i].Traits[j].Properties
propMap, err := util.RawExtension2Map(props)
if err != nil {
return
}
newPropMap := map[string]interface{}{}
for k := range propMap {
if k != "class" {
newPropMap[k] = propMap[k]
}
}
ext, err := json.Marshal(newPropMap)
if err != nil {
return
}
app.Spec.Components[i].Traits[j].Properties = &runtime.RawExtension{
Raw: ext,
}
}
}
}
}
test/e2e-test/suite_test.go
+27
View File
@@ -31,13 +31,17 @@ import (
. "github.com/onsi/gomega"
kruise "github.com/openkruise/kruise-api/apps/v1alpha1"
"github.com/pkg/errors"
networkv1 "k8s.io/api/networking/v1"
rbac "k8s.io/api/rbac/v1"
crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
k8sutils "k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
@@ -58,6 +62,10 @@ var manualscalertrait v1alpha2.TraitDefinition
var roleName = "oam-example-com"
var roleBindingName = "oam-role-binding"
var (
noNetworkingV1 bool
)
// A DefinitionExtension is an Object type for xxxDefinitin.spec.extension
type DefinitionExtension struct {
Alias string `json:"alias,omitempty"`
@@ -104,6 +112,8 @@ var _ = BeforeSuite(func(done Done) {
}
By("Finished setting up test environment")
detectAPIVersion()
// Create manual scaler trait definition
manualscalertrait = v1alpha2.TraitDefinition{
ObjectMeta: metav1.ObjectMeta{
@@ -219,3 +229,20 @@ func RequestReconcileNow(ctx context.Context, o client.Object) {
func randomNamespaceName(basic string) string {
return fmt.Sprintf("%s-%s", basic, strconv.FormatInt(rand.Int63(), 16))
}
// detectAPIVersion helps detect legacy GVK
func detectAPIVersion() {
err := k8sClient.Create(context.Background(), &networkv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "test-ingress",
},
Spec: networkv1.IngressSpec{
IngressClassName: k8sutils.StringPtr("nginx"),
Rules: []networkv1.IngressRule{},
},
})
var noKindMatchErr = &meta.NoKindMatchError{}
if err != nil && errors.As(err, &noKindMatchErr) {
noNetworkingV1 = true
}
}
test/e2e-test/testdata/app/app_healthscope.yaml
Vendored
+4 -2
View File
@@ -13,8 +13,9 @@ spec:
image: oamdev/testapp:v1
port: 8080
traits:
- type: ingress
- type: gateway
properties:
class: traefik
domain: test.my.domain
http:
"/": 8080
@@ -27,8 +28,9 @@ spec:
image: oamdev/testapp:v1
port: 8080
traits:
- type: ingress
- type: gateway
properties:
class: traefik
domain: test.my.domain
http:
"/": 8080
test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml
Vendored
+2 -1
View File
@@ -21,8 +21,9 @@ spec:
image: oamdev/testapp:v1
port: 8080
traits:
- type: ingress
- type: gateway
properties:
class: traefik
domain: test.my.domain
http:
"/": 8080