From 1125f87fd78b715a29efe3c0c1ff74b6d9292c56 Mon Sep 17 00:00:00 2001 From: qiaozp <47812250+chivalryq@users.noreply.github.com> Date: Fri, 22 Apr 2022 17:05:19 +0800 Subject: [PATCH] Feat: run e2e test on several K8s version (#3714) * Feat: run e2e test on several K8s version Signed-off-by: qiaozp * upgrade kind version Signed-off-by: qiaozp * rollback 1.22 Signed-off-by: qiaozp * bump up kruise Signed-off-by: qiaozp * with sha Signed-off-by: qiaozp * try fix ci Signed-off-by: qiaozp * use 1.22 Signed-off-by: qiaozp * remove sah Signed-off-by: qiaozp * revert kind Signed-off-by: qiaozp --- .github/workflows/apiserver-test.yaml | 36 ++++++++++++++++++--- .github/workflows/e2e-multicluster-test.yml | 35 ++++++++++++++++++-- .github/workflows/e2e-rollout-test.yml | 31 ++++++++++++++++-- .github/workflows/e2e-test.yml | 31 ++++++++++++++++-- .github/workflows/unit-test.yml | 4 +-- makefiles/const.mk | 2 +- makefiles/e2e.mk | 2 +- 7 files changed, 126 insertions(+), 15 deletions(-) diff --git a/.github/workflows/apiserver-test.yaml b/.github/workflows/apiserver-test.yaml index 601052d69..098677221 100644 --- a/.github/workflows/apiserver-test.yaml +++ b/.github/workflows/apiserver-test.yaml @@ -6,7 +6,7 @@ on: - master - release-* - apiserver - workflow_dispatch: {} + workflow_dispatch: { } pull_request: branches: - master @@ -18,6 +18,8 @@ env: GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' KIND_VERSION: 'v0.7.0' + KIND_IMAGE_VERSION: '[\"v1.20.7\"]' + KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]' jobs: @@ -35,10 +37,36 @@ jobs: do_not_skip: '["workflow_dispatch", "schedule", "push"]' concurrent_skipping: false + set-k8s-matrix: + runs-on: ubuntu-20.04 + outputs: + matrix: ${{ steps.set-k8s-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + - uses: mukunku/tag-exists-action@v1.0.0 + id: checkTag + with: + tag: 'v1' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - id: set-k8s-matrix + run: | + echo ${{ steps.checkTag.outputs.exists }} + if [ "${{ steps.checkTag.outputs.exists }}" = "true" ]; then + echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}" + else + echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}" + fi + + apiserver-unit-tests: runs-on: aliyun - needs: detect-noop + needs: [ detect-noop,set-k8s-matrix ] if: needs.detect-noop.outputs.noop != 'true' + strategy: + matrix: + k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }} steps: - name: Set up Go @@ -65,7 +93,7 @@ jobs: - name: Setup Kind Cluster (Worker) run: | kind delete cluster --name worker - kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca --name worker + kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker kubectl version kubectl cluster-info kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig @@ -74,7 +102,7 @@ jobs: - name: Setup Kind Cluster (Hub) run: | kind delete cluster - kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca + kind create cluster --image kindest/node:${{ matrix.k8s-version }} kubectl version kubectl cluster-info diff --git a/.github/workflows/e2e-multicluster-test.yml b/.github/workflows/e2e-multicluster-test.yml index 85e9fcc8c..89b3cdaf1 100644 --- a/.github/workflows/e2e-multicluster-test.yml +++ b/.github/workflows/e2e-multicluster-test.yml @@ -16,6 +16,8 @@ env: GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' KIND_VERSION: 'v0.7.0' + KIND_IMAGE_VERSION: '[\"v1.20.7\"]' + KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]' jobs: @@ -33,10 +35,37 @@ jobs: do_not_skip: '["workflow_dispatch", "schedule", "push"]' concurrent_skipping: false + set-k8s-matrix: + runs-on: ubuntu-20.04 + outputs: + matrix: ${{ steps.set-k8s-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + - uses: mukunku/tag-exists-action@v1.0.0 + id: checkTag + with: + tag: 'v1' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - id: set-k8s-matrix + run: | + echo ${{ steps.checkTag.outputs.exists }} + if [ "${{ steps.checkTag.outputs.exists }}" = "true" ]; then + echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}" + else + echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}" + fi + + e2e-multi-cluster-tests: runs-on: aliyun - needs: detect-noop + needs: [ detect-noop,set-k8s-matrix ] if: needs.detect-noop.outputs.noop != 'true' + strategy: + matrix: + k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }} + steps: - name: Check out code into the Go module directory @@ -60,7 +89,7 @@ jobs: - name: Setup Kind Cluster (Worker) run: | kind delete cluster --name worker - kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca --name worker + kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker kubectl version kubectl cluster-info kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig @@ -69,7 +98,7 @@ jobs: - name: Setup Kind Cluster (Hub) run: | kind delete cluster - kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca + kind create cluster --image kindest/node:${{ matrix.k8s-version }} kubectl version kubectl cluster-info diff --git a/.github/workflows/e2e-rollout-test.yml b/.github/workflows/e2e-rollout-test.yml index b5bfec9c1..2611ee139 100644 --- a/.github/workflows/e2e-rollout-test.yml +++ b/.github/workflows/e2e-rollout-test.yml @@ -16,6 +16,8 @@ env: GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' KIND_VERSION: 'v0.7.0' + KIND_IMAGE_VERSION: '[\"v1.20.7\"]' + KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]' jobs: @@ -33,10 +35,35 @@ jobs: do_not_skip: '["workflow_dispatch", "schedule", "push"]' concurrent_skipping: false + set-k8s-matrix: + runs-on: ubuntu-20.04 + outputs: + matrix: ${{ steps.set-k8s-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + - uses: mukunku/tag-exists-action@v1.0.0 + id: checkTag + with: + tag: 'v1' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - id: set-k8s-matrix + run: | + echo ${{ steps.checkTag.outputs.exists }} + if [ "${{ steps.checkTag.outputs.exists }}" = "true" ]; then + echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}" + else + echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}" + fi + e2e-rollout-tests: runs-on: aliyun - needs: detect-noop + needs: [ detect-noop,set-k8s-matrix ] if: needs.detect-noop.outputs.noop != 'true' + strategy: + matrix: + k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }} steps: - name: Check out code into the Go module directory @@ -60,7 +87,7 @@ jobs: - name: Setup Kind Cluster run: | kind delete cluster - kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca + kind create cluster --image kindest/node:${{ matrix.k8s-version }} kubectl version kubectl cluster-info diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 4a65f381b..019d70b95 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -16,6 +16,8 @@ env: GO_VERSION: '1.17' GOLANGCI_VERSION: 'v1.38' KIND_VERSION: 'v0.7.0' + KIND_IMAGE_VERSION: '[\"v1.20.7\"]' + KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]' jobs: @@ -33,10 +35,35 @@ jobs: do_not_skip: '["workflow_dispatch", "schedule", "push"]' concurrent_skipping: false + set-k8s-matrix: + runs-on: ubuntu-20.04 + outputs: + matrix: ${{ steps.set-k8s-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + - uses: mukunku/tag-exists-action@v1.0.0 + id: checkTag + with: + tag: 'v1' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - id: set-k8s-matrix + run: | + echo ${{ steps.checkTag.outputs.exists }} + if [ "${{ steps.checkTag.outputs.exists }}" = "true" ]; then + echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}" + else + echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}" + fi + e2e-tests: runs-on: aliyun - needs: detect-noop + needs: [ detect-noop,set-k8s-matrix ] if: needs.detect-noop.outputs.noop != 'true' + strategy: + matrix: + k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }} steps: - name: Check out code into the Go module directory @@ -60,7 +87,7 @@ jobs: - name: Setup Kind Cluster run: | kind delete cluster - kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca + kind create cluster --image kindest/node:${{ matrix.k8s-version }} kubectl version kubectl cluster-info diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index e62fe5812..bfa8e4746 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -58,7 +58,7 @@ jobs: restore-keys: ${{ runner.os }}-pkg- - name: Install ginkgo - run: | + run: | sudo apt-get install -y golang-ginkgo-dev - name: Setup Kind Cluster @@ -72,7 +72,7 @@ jobs: version: 3.1.0 kubebuilderOnly: false kubernetesVersion: v1.21.2 - + - name: Run Make test run: make test diff --git a/makefiles/const.mk b/makefiles/const.mk index 98eec739d..94f18b332 100644 --- a/makefiles/const.mk +++ b/makefiles/const.mk @@ -47,5 +47,5 @@ VELA_CORE_TEST_IMAGE ?= vela-core-test:$(GIT_COMMIT) VELA_APISERVER_IMAGE ?= apiserver:latest VELA_RUNTIME_ROLLOUT_IMAGE ?= vela-runtime-rollout:latest VELA_RUNTIME_ROLLOUT_TEST_IMAGE ?= vela-runtime-rollout-test:$(GIT_COMMIT) -RUNTIME_CLUSTER_CONFIG ?= /tmp/worker.kubeconfig +RUNTIME_CLUSTER_CONFIG ?= /tmp/worker.client.kubeconfig RUNTIME_CLUSTER_NAME ?= worker \ No newline at end of file diff --git a/makefiles/e2e.mk b/makefiles/e2e.mk index 8d1aa2ef7..95c2c912f 100644 --- a/makefiles/e2e.mk +++ b/makefiles/e2e.mk @@ -15,7 +15,7 @@ setup-runtime-e2e-cluster: .PHONY: e2e-setup e2e-setup: - helm install kruise https://github.com/openkruise/kruise/releases/download/v0.9.0/kruise-chart.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true" + helm install kruise https://github.com/openkruise/charts/releases/download/kruise-1.1.0/kruise-1.1.0.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true" sh ./hack/e2e/modify_charts.sh helm upgrade --install --create-namespace --namespace vela-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set applicationRevisionLimit=5 --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --wait kubevela ./charts/vela-core helm upgrade --install --create-namespace --namespace oam-runtime-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --wait oam-runtime ./charts/oam-runtime