Feat: run e2e test on several K8s version (#3714)

* Feat: run e2e test on several K8s version Signed-off-by: qiaozp <chivalry.pp@gmail.com> * upgrade kind version Signed-off-by: qiaozp <chivalry.pp@gmail.com> * rollback 1.22 Signed-off-by: qiaozp <chivalry.pp@gmail.com> * bump up kruise Signed-off-by: qiaozp <chivalry.pp@gmail.com> * with sha Signed-off-by: qiaozp <chivalry.pp@gmail.com> * try fix ci Signed-off-by: qiaozp <chivalry.pp@gmail.com> * use 1.22 Signed-off-by: qiaozp <chivalry.pp@gmail.com> * remove sah Signed-off-by: qiaozp <chivalry.pp@gmail.com> * revert kind Signed-off-by: qiaozp <chivalry.pp@gmail.com>
2026-02-14 10:00:06 +00:00 · 2022-04-22 17:05:19 +08:00
parent 520e67f63b
commit 1125f87fd7
7 changed files with 126 additions and 15 deletions
--- a/.github/workflows/apiserver-test.yaml
+++ b/.github/workflows/apiserver-test.yaml
@@ -6,7 +6,7 @@ on:
      - master
      - release-*
      - apiserver
-  workflow_dispatch: {}
+  workflow_dispatch: { }
  pull_request:
    branches:
      - master
@@ -18,6 +18,8 @@ env:
  GO_VERSION: '1.17'
  GOLANGCI_VERSION: 'v1.38'
  KIND_VERSION: 'v0.7.0'
+  KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
+  KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'

 jobs:

@@ -35,10 +37,36 @@ jobs:
          do_not_skip: '["workflow_dispatch", "schedule", "push"]'
          concurrent_skipping: false

+  set-k8s-matrix:
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.set-k8s-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: mukunku/tag-exists-action@v1.0.0
+        id: checkTag
+        with:
+          tag: 'v1'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: set-k8s-matrix
+        run: |
+          echo ${{ steps.checkTag.outputs.exists }}
+          if [ "${{ steps.checkTag.outputs.exists }}" = "true" ]; then
+            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
+          else
+            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
+          fi
+
+
  apiserver-unit-tests:
    runs-on: aliyun
-    needs: detect-noop
+    needs: [ detect-noop,set-k8s-matrix ]
    if: needs.detect-noop.outputs.noop != 'true'
+    strategy:
+      matrix:
+        k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }}

    steps:
      - name: Set up Go
@@ -65,7 +93,7 @@ jobs:
      - name: Setup Kind Cluster (Worker)
        run: |
          kind delete cluster --name worker
-          kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca --name worker
+          kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker
          kubectl version
          kubectl cluster-info
          kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig
@@ -74,7 +102,7 @@ jobs:
      - name: Setup Kind Cluster (Hub)
        run: |
          kind delete cluster
-          kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca
+          kind create cluster --image kindest/node:${{ matrix.k8s-version }}
          kubectl version
          kubectl cluster-info

--- a/.github/workflows/e2e-multicluster-test.yml
+++ b/.github/workflows/e2e-multicluster-test.yml
@@ -16,6 +16,8 @@ env:
  GO_VERSION: '1.17'
  GOLANGCI_VERSION: 'v1.38'
  KIND_VERSION: 'v0.7.0'
+  KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
+  KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'

 jobs:

@@ -33,10 +35,37 @@ jobs:
          do_not_skip: '["workflow_dispatch", "schedule", "push"]'
          concurrent_skipping: false

+  set-k8s-matrix:
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.set-k8s-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: mukunku/tag-exists-action@v1.0.0
+        id: checkTag
+        with:
+          tag: 'v1'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: set-k8s-matrix
+        run: |
+          echo ${{ steps.checkTag.outputs.exists }}
+          if [ "${{ steps.checkTag.outputs.exists }}" = "true" ]; then
+            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
+          else
+            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
+          fi
+
+
  e2e-multi-cluster-tests:
    runs-on: aliyun
-    needs: detect-noop
+    needs: [ detect-noop,set-k8s-matrix ]
    if: needs.detect-noop.outputs.noop != 'true'
+    strategy:
+      matrix:
+        k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }}
+

    steps:
      - name: Check out code into the Go module directory
@@ -60,7 +89,7 @@ jobs:
      - name: Setup Kind Cluster (Worker)
        run: |
          kind delete cluster --name worker
-          kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca --name worker
+          kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker
          kubectl version
          kubectl cluster-info
          kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig
@@ -69,7 +98,7 @@ jobs:
      - name: Setup Kind Cluster (Hub)
        run: |
          kind delete cluster
-          kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca
+          kind create cluster --image kindest/node:${{ matrix.k8s-version }}
          kubectl version
          kubectl cluster-info

--- a/.github/workflows/e2e-rollout-test.yml
+++ b/.github/workflows/e2e-rollout-test.yml
@@ -16,6 +16,8 @@ env:
  GO_VERSION: '1.17'
  GOLANGCI_VERSION: 'v1.38'
  KIND_VERSION: 'v0.7.0'
+  KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
+  KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'

 jobs:

@@ -33,10 +35,35 @@ jobs:
          do_not_skip: '["workflow_dispatch", "schedule", "push"]'
          concurrent_skipping: false

+  set-k8s-matrix:
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.set-k8s-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: mukunku/tag-exists-action@v1.0.0
+        id: checkTag
+        with:
+          tag: 'v1'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: set-k8s-matrix
+        run: |
+          echo ${{ steps.checkTag.outputs.exists }}
+          if [ "${{ steps.checkTag.outputs.exists }}" = "true" ]; then
+            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
+          else
+            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
+          fi
+
  e2e-rollout-tests:
    runs-on: aliyun
-    needs: detect-noop
+    needs: [ detect-noop,set-k8s-matrix ]
    if: needs.detect-noop.outputs.noop != 'true'
+    strategy:
+      matrix:
+        k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }}

    steps:
      - name: Check out code into the Go module directory
@@ -60,7 +87,7 @@ jobs:
      - name: Setup Kind Cluster
        run: |
          kind delete cluster
-          kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca
+          kind create cluster --image kindest/node:${{ matrix.k8s-version }}
          kubectl version
          kubectl cluster-info

--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -16,6 +16,8 @@ env:
  GO_VERSION: '1.17'
  GOLANGCI_VERSION: 'v1.38'
  KIND_VERSION: 'v0.7.0'
+  KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
+  KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'

 jobs:

@@ -33,10 +35,35 @@ jobs:
          do_not_skip: '["workflow_dispatch", "schedule", "push"]'
          concurrent_skipping: false

+  set-k8s-matrix:
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.set-k8s-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: mukunku/tag-exists-action@v1.0.0
+        id: checkTag
+        with:
+          tag: 'v1'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: set-k8s-matrix
+        run: |
+          echo ${{ steps.checkTag.outputs.exists }}
+          if [ "${{ steps.checkTag.outputs.exists }}" = "true" ]; then
+            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
+          else
+            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
+          fi
+
  e2e-tests:
    runs-on: aliyun
-    needs: detect-noop
+    needs: [ detect-noop,set-k8s-matrix ]
    if: needs.detect-noop.outputs.noop != 'true'
+    strategy:
+      matrix:
+        k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }}

    steps:
      - name: Check out code into the Go module directory
@@ -60,7 +87,7 @@ jobs:
      - name: Setup Kind Cluster
        run: |
          kind delete cluster
-          kind create cluster --image kindest/node:v1.20.7@sha256:688fba5ce6b825be62a7c7fe1415b35da2bdfbb5a69227c499ea4cc0008661ca
+          kind create cluster --image kindest/node:${{ matrix.k8s-version }}
          kubectl version
          kubectl cluster-info

--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -58,7 +58,7 @@ jobs:
          restore-keys: ${{ runner.os }}-pkg-

      - name: Install ginkgo
-        run:  |
+        run: |
          sudo apt-get install -y golang-ginkgo-dev

      - name: Setup Kind Cluster
@@ -72,7 +72,7 @@ jobs:
          version: 3.1.0
          kubebuilderOnly: false
          kubernetesVersion: v1.21.2
-      
+
      - name: Run Make test
        run: make test

--- a/makefiles/const.mk
+++ b/makefiles/const.mk
@@ -47,5 +47,5 @@ VELA_CORE_TEST_IMAGE ?= vela-core-test:$(GIT_COMMIT)
 VELA_APISERVER_IMAGE      ?= apiserver:latest
 VELA_RUNTIME_ROLLOUT_IMAGE       ?= vela-runtime-rollout:latest
 VELA_RUNTIME_ROLLOUT_TEST_IMAGE  ?= vela-runtime-rollout-test:$(GIT_COMMIT)
-RUNTIME_CLUSTER_CONFIG ?= /tmp/worker.kubeconfig
+RUNTIME_CLUSTER_CONFIG ?= /tmp/worker.client.kubeconfig
 RUNTIME_CLUSTER_NAME ?= worker
--- a/makefiles/e2e.mk
+++ b/makefiles/e2e.mk
@@ -15,7 +15,7 @@ setup-runtime-e2e-cluster:

 .PHONY: e2e-setup
 e2e-setup:
-	helm install kruise https://github.com/openkruise/kruise/releases/download/v0.9.0/kruise-chart.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true"
+	helm install kruise https://github.com/openkruise/charts/releases/download/kruise-1.1.0/kruise-1.1.0.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true"
 	sh ./hack/e2e/modify_charts.sh
 	helm upgrade --install --create-namespace --namespace vela-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set applicationRevisionLimit=5 --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --wait kubevela ./charts/vela-core
 	helm upgrade --install --create-namespace --namespace oam-runtime-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --wait oam-runtime ./charts/oam-runtime