fix enable addon cannot update definition bug (#4686)

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>
(cherry picked from commit b5cd806efa)

Co-authored-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

.github/workflows/apiserver-test.yaml

@@ -19,9 +19,8 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  KIND_VERSION: 'v0.7.0'
-  KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
-  KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'
+  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.23\"]'
 
 jobs:
 
@@ -48,9 +47,9 @@ jobs:
         run: |
           if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
             echo "pushing tag: ${{ github.ref_name }}"
-            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
+            echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}"
           else
-            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
+            echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}"
           fi
 
   apiserver-unit-tests:
@@ -111,6 +110,9 @@ jobs:
     strategy:
       matrix:
         k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.k8s-version }}
+      cancel-in-progress: true
 
     steps:
       - name: Set up Go
@@ -128,35 +130,50 @@ jobs:
         run: |
           go get -v -t -d ./...
 
-      - name: Setup Kind
-        uses: engineerd/setup-kind@v0.5.0
+      - name: Tear down K3d if exist
+        run: |
+          k3d cluster delete || true
+          k3d cluster delete worker || true
+
+      - name: Calculate K3d args
+        run: |
+          EGRESS_ARG=""
+          if [[ "${{ matrix.k8s-version }}" == v1.23 ]]; then
+            EGRESS_ARG="--k3s-arg --egress-selector-mode=disabled@server:0"
+          fi
+          echo "EGRESS_ARG=${EGRESS_ARG}" >> $GITHUB_ENV 
+
+      - name: Setup K3d (Hub)
+        uses: nolar/setup-k3d-k3s@v1.0.8
         with:
-          version: ${{ env.KIND_VERSION }}
-          skipClusterCreation: true
+          version: ${{ matrix.k8s-version }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          k3d-args: ${{ env.EGRESS_ARG }}
 
-      - name: Setup Kind Cluster (Worker)
+
+      - name: Setup K3d (Worker)
+        uses: nolar/setup-k3d-k3s@v1.0.8
+        with:
+          version: ${{ matrix.k8s-version }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          k3d-name: worker
+          k3d-args: --kubeconfig-update-default=false --network=k3d-k3s-default ${{ env.EGRESS_ARG }}
+
+
+      - name: Kind Cluster (Worker)
         run: |
-          kind delete cluster --name worker
-          kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker
-          kubectl version
-          kubectl cluster-info
-          kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig
-          kind get kubeconfig --name worker > /tmp/worker.client.kubeconfig
+          internal_ip=$(docker network inspect k3d-k3s-default|jq ".[0].Containers"| jq -r '.[]| select(.Name=="k3d-worker-server-0")|.IPv4Address' | cut -d/ -f1)
+          k3d kubeconfig get worker > /tmp/worker.client.kubeconfig
+          cp /tmp/worker.client.kubeconfig /tmp/worker.kubeconfig
+          sed -i "s/0.0.0.0:[0-9]\+/$internal_ip:6443/"  /tmp/worker.kubeconfig
 
-      - name: Setup Kind Cluster (Hub)
-        run: |
-          kind delete cluster
-          kind create cluster --image kindest/node:${{ matrix.k8s-version }}
-          kubectl version
-          kubectl cluster-info
-
-      - name: Load Image to kind cluster
-        run: make kind-load
+      - name: Load image to k3d cluster
+        run: make image-load
 
       - name: Cleanup for e2e tests
         run: |
-          make e2e-cleanup
           make vela-cli
+          make e2e-cleanup
           make e2e-setup-core
           bin/vela addon enable fluxcd
           timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $? -eq 0 ] ; then break; else sleep 5; fi;done'

.github/workflows/e2e-multicluster-test.yml

@@ -17,9 +17,8 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  KIND_VERSION: 'v0.7.0'
-  KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
-  KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'
+  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.23\"]'
 
 jobs:
 
@@ -46,9 +45,9 @@ jobs:
         run: |
           if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
             echo "pushing tag: ${{ github.ref_name }}"
-            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
+            echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}"
           else
-            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
+            echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}"
           fi
 
 
@@ -59,6 +58,9 @@ jobs:
     strategy:
       matrix:
         k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.k8s-version }}
+      cancel-in-progress: true
 
 
     steps:
@@ -74,37 +76,49 @@ jobs:
         run: |
           go get -v -t -d ./...
 
-      - name: Setup Kind
-        uses: engineerd/setup-kind@v0.5.0
+      - name: Tear down K3d if exist
+        run: |
+          k3d cluster delete || true
+          k3d cluster delete worker || true
+
+      - name: Calculate K3d args
+        run: |
+          EGRESS_ARG=""
+          if [[ "${{ matrix.k8s-version }}" == v1.23 ]]; then
+            EGRESS_ARG="--k3s-arg --egress-selector-mode=disabled@server:0"
+          fi
+          echo "EGRESS_ARG=${EGRESS_ARG}" >> $GITHUB_ENV 
+
+      - name: Setup K3d (Hub)
+        uses: nolar/setup-k3d-k3s@v1.0.8
         with:
-          version: ${{ env.KIND_VERSION }}
-          skipClusterCreation: true
+          version: ${{ matrix.k8s-version }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          k3d-args: ${{ env.EGRESS_ARG }}
 
-      - name: Setup Kind Cluster (Worker)
+      - name: Setup K3d (Worker)
+        uses: nolar/setup-k3d-k3s@v1.0.8
+        with:
+          version: ${{ matrix.k8s-version }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          k3d-name: worker
+          k3d-args: --kubeconfig-update-default=false --network=k3d-k3s-default ${{ env.EGRESS_ARG }}
+
+      - name: Generating internal worker kubeconfig
         run: |
-          kind delete cluster --name worker
-          kind create cluster --image kindest/node:${{ matrix.k8s-version }} --name worker
-          kubectl version
-          kubectl cluster-info
-          kind get kubeconfig --name worker --internal > /tmp/worker.kubeconfig
-          kind get kubeconfig --name worker > /tmp/worker.client.kubeconfig
+          internal_ip=$(docker network inspect k3d-k3s-default|jq ".[0].Containers"| jq -r '.[]| select(.Name=="k3d-worker-server-0")|.IPv4Address' | cut -d/ -f1)
+          k3d kubeconfig get worker > /tmp/worker.client.kubeconfig
+          cp /tmp/worker.client.kubeconfig /tmp/worker.kubeconfig
+          sed -i "s/0.0.0.0:[0-9]\+/$internal_ip:6443/"  /tmp/worker.kubeconfig
 
-      - name: Setup Kind Cluster (Hub)
-        run: |
-          kind delete cluster
-          kind create cluster --image kindest/node:${{ matrix.k8s-version }}
-          kubectl version
-          kubectl cluster-info
-
-      - name: Load Image to kind cluster (Hub)
-        run: make kind-load
+      - name: Load image to k3d cluster (hub and worker)
+        run: make image-load image-load-runtime-cluster
 
       - name: Cleanup for e2e tests
         run: |
-          make e2e-cleanup
           make vela-cli
+          make e2e-cleanup
           make e2e-setup-core-auth
-          make
           make setup-runtime-e2e-cluster
 
       - name: Run e2e multicluster tests

.github/workflows/e2e-rollout-test.yml

@@ -17,9 +17,8 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  KIND_VERSION: 'v0.7.0'
-  KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
-  KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'
+  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.23\"]'
 
 jobs:
 
@@ -46,9 +45,9 @@ jobs:
         run: |
           if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
             echo "pushing tag: ${{ github.ref_name }}"
-            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
+            echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}"
           else
-            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
+            echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}"
           fi
 
   e2e-rollout-tests:
@@ -58,6 +57,10 @@ jobs:
     strategy:
       matrix:
         k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.k8s-version }}
+      cancel-in-progress: true
+
 
     steps:
       - name: Check out code into the Go module directory
@@ -72,33 +75,35 @@ jobs:
         run: |
           go get -v -t -d ./...
 
-      - name: Setup Kind
-        uses: engineerd/setup-kind@v0.5.0
-        with:
-          version: ${{ env.KIND_VERSION }}
-          skipClusterCreation: true
-
-      - name: Setup Kind Cluster
+      - name: Tear down K3d if exist
         run: |
-          kind delete cluster
-          kind create cluster --image kindest/node:${{ matrix.k8s-version }}
-          kubectl version
-          kubectl cluster-info
+          k3d cluster delete || true
+          k3d cluster delete worker || true
 
-      - name: Load Image to kind cluster
-        run: make kind-load
+      - name: Calculate K3d args
+        run: |
+          EGRESS_ARG=""
+          if [[ "${{ matrix.k8s-version }}" == v1.23 ]]; then
+            EGRESS_ARG="--k3s-arg --egress-selector-mode=disabled@server:0"
+          fi
+          echo "EGRESS_ARG=${EGRESS_ARG}" >> $GITHUB_ENV 
 
-      - name: Run Make
-        run: make
+      - name: Setup K3d
+        uses: nolar/setup-k3d-k3s@v1.0.8
+        with:
+          version: ${{ matrix.k8s-version }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          k3d-args: ${{ env.EGRESS_ARG }}
 
-      - name: Run Make Manager
-        run: make manager
+      - name: Load image to k3d cluster
+        run: make image-load image-load-runtime-cluster
 
       - name: Prepare for e2e tests
         run: |
+          make vela-cli
           make e2e-cleanup
-          make e2e-setup
-          helm lint ./charts/vela-core
+          make e2e-setup-core
+          make setup-runtime-e2e-cluster
           helm test -n vela-system kubevela --timeout 5m
 
       - name: Run e2e tests

.github/workflows/e2e-test.yml

@@ -17,9 +17,8 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  KIND_VERSION: 'v0.7.0'
-  KIND_IMAGE_VERSION: '[\"v1.20.7\"]'
-  KIND_IMAGE_VERSIONS: '[\"v1.18.20\",\"v1.20.7\",\"v1.22.7\"]'
+  K3D_IMAGE_VERSION: '[\"v1.20\",\"v1.23\"]'
+  K3D_IMAGE_VERSIONS: '[\"v1.20\",\"v1.23\"]'
 
 jobs:
 
@@ -46,9 +45,9 @@ jobs:
         run: |
           if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
             echo "pushing tag: ${{ github.ref_name }}"
-            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSIONS }}"
+            echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSIONS }}"
           else
-            echo "::set-output name=matrix::${{ env.KIND_IMAGE_VERSION }}"
+            echo "::set-output name=matrix::${{ env.K3D_IMAGE_VERSION }}"
           fi
 
   e2e-tests:
@@ -58,6 +57,10 @@ jobs:
     strategy:
       matrix:
         k8s-version: ${{ fromJson(needs.set-k8s-matrix.outputs.matrix) }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.k8s-version }}
+      cancel-in-progress: true
+
 
     steps:
       - name: Check out code into the Go module directory
@@ -72,33 +75,36 @@ jobs:
         run: |
           go get -v -t -d ./...
 
-      - name: Setup Kind
-        uses: engineerd/setup-kind@v0.5.0
-        with:
-          version: ${{ env.KIND_VERSION }}
-          skipClusterCreation: true
-
-      - name: Setup Kind Cluster
+      - name: Tear down K3d if exist
         run: |
-          kind delete cluster
-          kind create cluster --image kindest/node:${{ matrix.k8s-version }}
-          kubectl version
-          kubectl cluster-info
+          k3d cluster delete || true
+          k3d cluster delete worker || true
 
-      - name: Load Image to kind cluster
-        run: make kind-load
+      - name: Calculate K3d args
+        run: |
+          EGRESS_ARG=""
+          if [[ "${{ matrix.k8s-version }}" == v1.23 ]]; then
+            EGRESS_ARG="--k3s-arg --egress-selector-mode=disabled@server:0"
+          fi
+          echo "EGRESS_ARG=${EGRESS_ARG}" >> $GITHUB_ENV 
+
+      - name: Setup K3d
+        uses: nolar/setup-k3d-k3s@v1.0.8
+        with:
+          version: ${{ matrix.k8s-version }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          k3d-args: ${{ env.EGRESS_ARG }}
+
+      - name: Load image to k3d cluster
+        run: make image-load
 
       - name: Run Make
         run: make
 
-      - name: Run Make Manager
-        run: make manager
-
       - name: Prepare for e2e tests
         run: |
           make e2e-cleanup
-          make e2e-setup
-          helm lint ./charts/vela-core
+          make e2e-setup-core
           helm test -n vela-system kubevela --timeout 5m
 
       - name: Run api e2e tests

.github/workflows/go.yml

@@ -15,7 +15,6 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  KIND_VERSION: 'v0.7.0'
 
 jobs:
 

.github/workflows/unit-test.yml

@@ -15,7 +15,6 @@ env:
   # Common versions
   GO_VERSION: '1.17'
   GOLANGCI_VERSION: 'v1.38'
-  KIND_VERSION: 'v0.7.0'
 
 jobs:
 
@@ -61,10 +60,11 @@ jobs:
         run: |
           sudo apt-get install -y golang-ginkgo-dev
 
-      - name: Setup Kind Cluster
-        uses: engineerd/setup-kind@v0.5.0
+      - name: Setup K3d
+        uses: nolar/setup-k3d-k3s@v1.0.8
         with:
-          version: ${{ env.KIND_VERSION }}
+          version: v1.20
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: install Kubebuilder
         uses: RyanSiu1995/kubebuilder-action@v1.2

Makefile

@@ -82,17 +82,17 @@ endif
 
 
 
-# load docker image to the kind cluster
-kind-load: kind-load-runtime-cluster
+# load docker image to the k3d cluster
+image-load:
 	docker build -t $(VELA_CORE_TEST_IMAGE) -f Dockerfile.e2e .
-	kind load docker-image $(VELA_CORE_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_CORE_TEST_IMAGE)"; exit 1; }
+	k3d image import $(VELA_CORE_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_CORE_TEST_IMAGE)"; exit 1; }
 
-kind-load-runtime-cluster:
+image-load-runtime-cluster:
 	/bin/sh hack/e2e/build_runtime_rollout.sh
 	docker build -t $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) -f runtime/rollout/e2e/Dockerfile.e2e runtime/rollout/e2e/
 	rm -rf runtime/rollout/e2e/tmp
-	kind load docker-image $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)  || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; }
-	kind load docker-image $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) --name=$(RUNTIME_CLUSTER_NAME)  || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; }
+	k3d image import $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)  || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; }
+	k3d cluster get $(RUNTIME_CLUSTER_NAME) && k3d image import $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) --cluster=$(RUNTIME_CLUSTER_NAME) || echo "no worker cluster"
 
 # Run tests
 core-test: fmt vet manifests

charts/vela-core/templates/defwithtemplate/topology.yaml

@@ -13,7 +13,7 @@ spec:
       template: |
         parameter: {
         	// +usage=Specify the names of the clusters to select.
-        	cluster?: [...string]
+        	clusters?: [...string]
         	// +usage=Specify the label selector for clusters
         	clusterLabelSelector?: [string]: string
         	// +usage=Deprecated: Use clusterLabelSelector instead.

charts/vela-core/templates/test/test-application.yaml

@@ -13,12 +13,6 @@ spec:
       properties:
         image: {{ .Values.imageRegistry }}{{ .Values.test.app.repository }}:{{ .Values.test.app.tag }}
         port: 8000
-      traits:
-        - type: ingress
-          properties:
-            domain: testsvc.example.com
-            http:
-              "/": 8000
 ---
 apiVersion: v1
 kind: Pod
@@ -52,13 +46,5 @@ spec:
           kubectl -n {{ include "systemDefinitionNamespace" . }} wait --for=condition=available deployments helm-test-express-server --timeout 3m
           echo "deployment being available"
 
-          # wait for ingress being created
-          while ! [ `kubectl -n {{ include "systemDefinitionNamespace" . }} get ing helm-test-express-server | grep -v NAME | wc -l` = 1 ]; do
-            echo "waiting for ingress being created"
-            sleep 1
-          done
-
-
-
           echo "Application and its components are created"
   restartPolicy: Never

charts/vela-minimal/templates/defwithtemplate/topology.yaml

@@ -13,7 +13,7 @@ spec:
       template: |
         parameter: {
         	// +usage=Specify the names of the clusters to select.
-        	cluster?: [...string]
+        	clusters?: [...string]
         	// +usage=Specify the label selector for clusters
         	clusterLabelSelector?: [string]: string
         	// +usage=Deprecated: Use clusterLabelSelector instead.

e2e/application/application_test.go

@@ -294,7 +294,7 @@ var VelaQLPodListContext = func(context string, velaQL string) bool {
 					gomega.Expect(v.Status.Phase).To(gomega.ContainSubstring("Running"))
 				}
 				if v.Status.NodeName != "" {
-					gomega.Expect(v.Status.NodeName).To(gomega.ContainSubstring("kind-control-plane"))
+					gomega.Expect(v.Status.NodeName).To(gomega.ContainSubstring("k3d-k3s-default-server-0"))
 				}
 				if v.Metadata.Namespace != "" {
 					gomega.Expect(v.Metadata.Namespace).To(gomega.ContainSubstring("default"))

e2e/setup/setup_suite_test.go

@@ -1,35 +0,0 @@
-/*
-Copyright 2021 The KubeVela Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e
-
-import (
-	"testing"
-
-	"github.com/oam-dev/kubevela/e2e"
-
-	"github.com/onsi/ginkgo"
-	"github.com/onsi/gomega"
-)
-
-var _ = ginkgo.BeforeSuite(func() {
-	e2e.BeforeSuit()
-}, 30)
-
-func TestApplication(t *testing.T) {
-	gomega.RegisterFailHandler(ginkgo.Fail)
-	ginkgo.RunSpecs(t, "Setup Suite")
-}

e2e/setup/setup_test.go

@@ -1,25 +0,0 @@
-/*
-Copyright 2021 The KubeVela Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package e2e
-
-import (
-	"github.com/onsi/ginkgo"
-)
-
-var _ = ginkgo.Describe("Setup", func() {
-
-})

hack/e2e/end_e2e.sh

@@ -1,20 +1,20 @@
 . ./hack/e2e/end_e2e_core.sh
 
-OAM_CONTAINER_ID=$(docker exec kind-control-plane crictl ps | grep oam-runtime | grep --regexp  '^.............' -o)
-OAM_DOCKER_DIR=$(docker exec kind-control-plane crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${OAM_CONTAINER_ID}")
+OAM_CONTAINER_ID=$(docker exec k3d-k3s-default-server-0 crictl ps | grep oam-runtime | grep --regexp  '^.............' -o)
+OAM_DOCKER_DIR=$(docker exec k3d-k3s-default-server-0 crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${OAM_CONTAINER_ID}")
 echo "${OAM_CONTAINER_ID}"
 echo "${OAM_DOCKER_DIR}"
 
-docker exec kind-control-plane crictl exec "${OAM_CONTAINER_ID}" kill -2 1
+docker exec k3d-k3s-default-server-0 crictl exec "${OAM_CONTAINER_ID}" kill -2 1
 
 file=$OAM_DOCKER_DIR/e2e-profile.out
 echo "$file"
 n=1
 while [ $n -le 60 ];do
-    if_exist=$(docker exec kind-control-plane sh -c "test -f $file && echo 'ok'")
+    if_exist=$(docker exec k3d-k3s-default-server-0 sh -c "test -f $file && echo 'ok'")
     echo "$if_exist"
     if [ -n  "$if_exist" ];then
-        docker exec kind-control-plane cat "$file" > /tmp/oam-e2e-profile.out
+        docker exec k3d-k3s-default-server-0 cat "$file" > /tmp/oam-e2e-profile.out
         break
     fi
     echo file not generated yet

hack/e2e/end_e2e_core.sh

@@ -1,18 +1,18 @@
-CONTAINER_ID=$(docker exec kind-control-plane crictl ps | grep 'kubevela\s' | grep --regexp  '^.............' -o)
-DOCKER_DIR=$(docker exec kind-control-plane crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${CONTAINER_ID}")
+CONTAINER_ID=$(docker exec k3d-k3s-default-server-0 crictl ps | grep 'kubevela\s' | grep --regexp  '^.............' -o)
+DOCKER_DIR=$(docker exec k3d-k3s-default-server-0 crictl inspect --output go-template --template '{{range .info.runtimeSpec.mounts}}{{if (eq .destination "/workspace/data")}}{{.source}}{{end}}{{end}}' "${CONTAINER_ID}")
 echo "${CONTAINER_ID}"
 echo "${DOCKER_DIR}"
 
-docker exec kind-control-plane crictl exec "${CONTAINER_ID}" kill -2 1
+docker exec k3d-k3s-default-server-0 crictl exec "${CONTAINER_ID}" kill -2 1
 
 file=$DOCKER_DIR/e2e-profile.out
 echo "$file"
 n=1
 while [ $n -le 60 ];do
-    if_exist=$(docker exec kind-control-plane sh -c "test -f $file && echo 'ok'")
+    if_exist=$(docker exec k3d-k3s-default-server-0 sh -c "test -f $file && echo 'ok'")
     echo "$if_exist"
     if [ -n  "$if_exist" ];then
-        docker exec kind-control-plane cat "$file" > /tmp/e2e-profile.out
+        docker exec k3d-k3s-default-server-0 cat "$file" > /tmp/e2e-profile.out
         break
     fi
     echo file not generated yet

hack/pressure-test/tolerate-hollow-node.cue

@@ -9,9 +9,9 @@
 template: {
 	patch: spec: template: spec: {
 		tolerations: [{
-			key: "node.kubernetes.io/network-unavailable"
+			key:      "node.kubernetes.io/network-unavailable"
 			operator: "Exists"
-			effect: "NoSchedule"
+			effect:   "NoSchedule"
 		}]
 	}
 	parameter: {}

makefiles/e2e.mk

@@ -5,11 +5,18 @@ e2e-setup-core-pre-hook:
 .PHONY: e2e-setup-core-post-hook
 e2e-setup-core-post-hook:
 	kubectl wait --for=condition=Available deployment/kubevela-vela-core -n vela-system --timeout=180s
-	helm upgrade --install --namespace vela-system --wait oam-rollout --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) ./runtime/rollout/charts
+	helm install kruise https://github.com/openkruise/charts/releases/download/kruise-1.1.0/kruise-1.1.0.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true" --set daemon.socketLocation=/run/k3s/containerd/
 	go run ./e2e/addon/mock &
 	sleep 15
 	bin/vela addon enable ./e2e/addon/mock/testdata/fluxcd
 	bin/vela addon enable ./e2e/addon/mock/testdata/rollout
+	bin/vela addon enable ./e2e/addon/mock/testdata/terraform
+	bin/vela addon enable ./e2e/addon/mock/testdata/terraform-alibaba ALICLOUD_ACCESS_KEY=xxx ALICLOUD_SECRET_KEY=yyy ALICLOUD_REGION=cn-beijing
+
+	timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $$? -eq 0 ] ; then break; else sleep 5; fi;done'
+	kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s
+	kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s
+	kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s
 
 .PHONY: e2e-setup-core-wo-auth
 e2e-setup-core-wo-auth:
@@ -27,28 +34,26 @@ e2e-setup-core-auth: e2e-setup-core-pre-hook e2e-setup-core-w-auth e2e-setup-cor
 
 .PHONY: setup-runtime-e2e-cluster
 setup-runtime-e2e-cluster:
-	helm upgrade --install --create-namespace --namespace vela-system --kubeconfig=$(RUNTIME_CLUSTER_CONFIG) --set image.pullPolicy=IfNotPresent --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) --wait vela-rollout ./runtime/rollout/charts
+	helm upgrade --install                               \
+	    --namespace vela-system                          \
+	    --wait oam-rollout                               \
+	    --set image.repository=vela-runtime-rollout-test \
+	    --set image.tag=$(GIT_COMMIT)                    \
+	    ./runtime/rollout/charts
 
-.PHONY: e2e-setup
-e2e-setup:
-	helm install kruise https://github.com/openkruise/charts/releases/download/kruise-1.1.0/kruise-1.1.0.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true"
-	sh ./hack/e2e/modify_charts.sh
-	helm upgrade --install --create-namespace --namespace vela-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set applicationRevisionLimit=5 --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --wait kubevela ./charts/vela-core
-	helm upgrade --install --namespace vela-system --wait oam-rollout --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) ./runtime/rollout/charts
+	k3d cluster get $(RUNTIME_CLUSTER_NAME) && 			 \
+	helm upgrade --install                               \
+	    --create-namespace                               \
+	    --namespace vela-system                          \
+	    --kubeconfig=$(RUNTIME_CLUSTER_CONFIG)           \
+	    --set image.pullPolicy=IfNotPresent              \
+	    --set image.repository=vela-runtime-rollout-test \
+	    --set image.tag=$(GIT_COMMIT)                    \
+	    --wait vela-rollout                              \
+	    ./runtime/rollout/charts ||						 \
+	echo "no worker cluster"					   		 \
 
-	go run ./e2e/addon/mock &
-	sleep 15
-	bin/vela addon enable ./e2e/addon/mock/testdata/fluxcd
-	bin/vela addon enable ./e2e/addon/mock/testdata/terraform
-	bin/vela addon enable ./e2e/addon/mock/testdata/terraform-alibaba ALICLOUD_ACCESS_KEY=xxx ALICLOUD_SECRET_KEY=yyy ALICLOUD_REGION=cn-beijing
-	bin/vela addon enable ./e2e/addon/mock/testdata/rollout
-	ginkgo version
-	ginkgo -v -r e2e/setup
 
-	timeout 600s bash -c -- 'while true; do kubectl get ns flux-system; if [ $$? -eq 0 ] ; then break; else sleep 5; fi;done'
-	kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=vela-core,app.kubernetes.io/instance=kubevela -n vela-system --timeout=600s
-	kubectl wait --for=condition=Ready pod -l app=source-controller -n flux-system --timeout=600s
-	kubectl wait --for=condition=Ready pod -l app=helm-controller -n flux-system --timeout=600s
 
 .PHONY: e2e-api-test
 e2e-api-test:

pkg/addon/addon_suite_test.go

@@ -418,9 +418,12 @@ var _ = Describe("test override defs of addon", func() {
 		u := unstructured.Unstructured{Object: compUnstructured}
 		u.SetAPIVersion(v1beta1.SchemeGroupVersion.String())
 		u.SetKind(v1beta1.ComponentDefinitionKind)
+		u.SetLabels(map[string]string{"testUpdateLabel": "test"})
 		c, err := checkConflictDefs(ctx, k8sClient, []*unstructured.Unstructured{&u}, app.GetName())
 		Expect(err).Should(BeNil())
 		Expect(len(c)).Should(BeEquivalentTo(1))
+		// guarantee checkConflictDefs won't change source definition
+		Expect(u.GetLabels()["testUpdateLabel"]).Should(BeEquivalentTo("test"))
 
 		u.SetName("rollout")
 		c, err = checkConflictDefs(ctx, k8sClient, []*unstructured.Unstructured{&u}, app.GetName())

pkg/addon/utils.go

@@ -448,20 +448,21 @@ func isErrorCueRenderPathNotFound(err error, path string) bool {
 func checkConflictDefs(ctx context.Context, k8sClient client.Client, defs []*unstructured.Unstructured, appName string) (map[string]string, error) {
 	res := map[string]string{}
 	for _, def := range defs {
-		err := k8sClient.Get(ctx, client.ObjectKeyFromObject(def), def)
+		checkDef := def.DeepCopy()
+		err := k8sClient.Get(ctx, client.ObjectKeyFromObject(checkDef), checkDef)
 		if err == nil {
-			owner := metav1.GetControllerOf(def)
+			owner := metav1.GetControllerOf(checkDef)
 			if owner == nil || owner.Kind != v1beta1.ApplicationKind {
-				res[def.GetName()] = fmt.Sprintf("definition: %s already exist and not belong to any addon \n", def.GetName())
+				res[checkDef.GetName()] = fmt.Sprintf("definition: %s already exist and not belong to any addon \n", checkDef.GetName())
 				continue
 			}
 			if owner.Name != appName {
 				// if addon not belong to an addon or addon name is another one, we should put them in result
-				res[def.GetName()] = fmt.Sprintf("definition: %s in this addon already exist in %s \n", def.GetName(), addon.AppName2Addon(appName))
+				res[checkDef.GetName()] = fmt.Sprintf("definition: %s in this addon already exist in %s \n", checkDef.GetName(), addon.AppName2Addon(appName))
 			}
 		}
 		if err != nil && !errors2.IsNotFound(err) {
-			return nil, errors.Wrapf(err, "check definition %s", def.GetName())
+			return nil, errors.Wrapf(err, "check definition %s", checkDef.GetName())
 		}
 	}
 	return res, nil

pkg/apiserver/domain/model/system_info.go

@@ -32,6 +32,7 @@ const (
 // SystemInfo systemInfo model
 type SystemInfo struct {
 	BaseModel
+	SignedKey                   string        `json:"signedKey"`
 	InstallID                   string        `json:"installID"`
 	EnableCollection            bool          `json:"enableCollection"`
 	StatisticInfo               StatisticInfo `json:"statisticInfo,omitempty"`

pkg/apiserver/domain/service/authentication.go

@@ -58,7 +58,8 @@ const (
 	GrantTypeRefresh = "refresh"
 )
 
-var signedKey = ""
+// signedKey is the signed key of JWT
+var signedKey string
 
 // AuthenticationService is the service of authentication
 type AuthenticationService interface {

pkg/apiserver/domain/service/system_info.go

@@ -63,6 +63,7 @@ func (u systemInfoServiceImpl) Get(ctx context.Context) (*model.SystemInfo, erro
 		}
 		return info, nil
 	}
+	info.SignedKey = rand.String(32)
 	installID := rand.String(16)
 	info.InstallID = installID
 	info.EnableCollection = true
@@ -161,7 +162,7 @@ func (u systemInfoServiceImpl) Init(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	signedKey = info.InstallID
+	signedKey = info.SignedKey
 	_, err = initDexConfig(ctx, u.KubeClient, "http://velaux.com")
 	return err
 }

pkg/cue/model/sets/operation.go

@@ -22,6 +22,7 @@ import (
 
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/cue/format"
 	"cuelang.org/go/cue/parser"
 	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/pkg/errors"
@@ -361,7 +362,7 @@ func jsonMergePatch(base cue.Value, patch cue.Value) (string, error) {
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to merge base value and patch value by JsonMergePatch")
 	}
-	output, err := OpenBaiscLit(string(merged))
+	output, err := openJSON(string(merged))
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to parse open basic lit for merged result")
 	}
@@ -386,9 +387,47 @@ func jsonPatch(base cue.Value, patch cue.Value) (string, error) {
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to apply json patch")
 	}
-	output, err := OpenBaiscLit(string(merged))
+	output, err := openJSON(string(merged))
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to parse open basic lit for merged result")
 	}
 	return output, nil
 }
+
+func isEllipsis(elt ast.Node) bool {
+	_, ok := elt.(*ast.Ellipsis)
+	return ok
+}
+
+func openJSON(data string) (string, error) {
+	f, err := parser.ParseFile("-", data, parser.ParseComments)
+	if err != nil {
+		return "", err
+	}
+	ast.Walk(f, func(node ast.Node) bool {
+		field, ok := node.(*ast.Field)
+		if ok {
+			v := field.Value
+			switch lit := v.(type) {
+			case *ast.StructLit:
+				if len(lit.Elts) == 0 || !isEllipsis(lit.Elts[len(lit.Elts)-1]) {
+					lit.Elts = append(lit.Elts, &ast.Ellipsis{})
+				}
+			case *ast.ListLit:
+				if len(lit.Elts) == 0 || !isEllipsis(lit.Elts[len(lit.Elts)-1]) {
+					lit.Elts = append(lit.Elts, &ast.Ellipsis{})
+				}
+			}
+		}
+		return true
+	}, nil)
+	if len(f.Decls) > 0 {
+		if emb, ok := f.Decls[0].(*ast.EmbedDecl); ok {
+			if s, _ok := emb.Expr.(*ast.StructLit); _ok {
+				f.Decls = s.Elts
+			}
+		}
+	}
+	b, err := format.Node(f)
+	return string(b), err
+}

pkg/cue/model/sets/utils.go

@@ -182,7 +182,8 @@ func peelCloseExpr(node ast.Node) ast.Node {
 
 func lookField(node ast.Node, key string) ast.Node {
 	if field, ok := node.(*ast.Field); ok {
-		if labelStr(field.Label) == key {
+		// Note: the trim here has side effect: "\(v)" will be trimmed to \(v), only used for comparing fields
+		if strings.Trim(labelStr(field.Label), `"`) == strings.Trim(key, `"`) {
 			return field.Value
 		}
 	}

pkg/resourcekeeper/gc.go

@@ -352,7 +352,7 @@ func (h *gcHandler) deleteManagedResource(ctx context.Context, mr v1beta1.Manage
 				delete(labels, oam.LabelAppNamespace)
 				entry.obj.SetLabels(labels)
 			}
-			return errors.Wrapf(h.Client.Update(ctx, entry.obj), "failed to remove owner labels for resource while skipping gc")
+			return errors.Wrapf(h.Client.Update(_ctx, entry.obj), "failed to remove owner labels for resource while skipping gc")
 		}
 		if err := h.Client.Delete(_ctx, entry.obj); err != nil && !kerrors.IsNotFound(err) {
 			return errors.Wrapf(err, "failed to delete resource %s", mr.ResourceKey())

references/cli/debug.go

@@ -118,13 +118,20 @@ func (d *debugOpts) debugApplication(ctx context.Context, c common.Args, app *v1
 		}
 
 		// debug workflow steps
-		rawValue, err := d.getDebugRawValue(ctx, cli, pd, app)
+		rawValue, data, err := d.getDebugRawValue(ctx, cli, pd, app)
 		if err != nil {
+			if data != "" {
+				ioStreams.Info(color.RedString("%s%s", emojiFail, err.Error()))
+				ioStreams.Info(color.GreenString("Original Data in Debug:\n"), data)
+				return nil
+			}
 			return err
 		}
 
 		if err := d.handleCueSteps(rawValue, ioStreams); err != nil {
-			return err
+			ioStreams.Info(color.RedString("%s%s", emojiFail, err.Error()))
+			ioStreams.Info(color.GreenString("Original Data in Debug:\n"), data)
+			return nil
 		}
 	} else {
 		// dry run components
@@ -252,20 +259,20 @@ func unwrapStepName(step string) string {
 	return step
 }
 
-func (d *debugOpts) getDebugRawValue(ctx context.Context, cli client.Client, pd *packages.PackageDiscover, app *v1beta1.Application) (*value.Value, error) {
+func (d *debugOpts) getDebugRawValue(ctx context.Context, cli client.Client, pd *packages.PackageDiscover, app *v1beta1.Application) (*value.Value, string, error) {
 	debugCM := &corev1.ConfigMap{}
 	if err := cli.Get(ctx, client.ObjectKey{Name: debug.GenerateContextName(app.Name, d.step), Namespace: app.Namespace}, debugCM); err != nil {
-		return nil, fmt.Errorf("failed to get debug configmap, please make sure your application have the debug policy, you can add the debug policy by using `vela up -f <app.yaml> --debug`: %w", err)
+		return nil, "", fmt.Errorf("failed to get debug configmap, please make sure your application have the debug policy, you can add the debug policy by using `vela up -f <app.yaml> --debug`: %w", err)
 	}
 
 	if debugCM.Data == nil || debugCM.Data["debug"] == "" {
-		return nil, fmt.Errorf("debug configmap is empty")
+		return nil, "", fmt.Errorf("debug configmap is empty")
 	}
 	v, err := value.NewValue(debugCM.Data["debug"], pd, "")
 	if err != nil {
-		return nil, fmt.Errorf("failed to parse debug configmap: %w", err)
+		return nil, debugCM.Data["debug"], fmt.Errorf("failed to parse debug configmap: %w", err)
 	}
-	return v, nil
+	return v, debugCM.Data["debug"], nil
 }
 
 func (d *debugOpts) handleCueSteps(v *value.Value, ioStreams cmdutil.IOStreams) error {

references/cli/debug_test.go

@@ -93,9 +93,7 @@ func TestDebugApplicationWithWorkflow(t *testing.T) {
 					"debug": "error",
 				},
 			},
-			step:        "test-wf1",
-			focus:       "test",
-			expectedErr: "failed to parse debug configmap",
+			step: "test-wf1",
 		},
 		"success": {
 			app: &v1beta1.Application{

references/cli/def.go

@@ -289,7 +289,7 @@ func NewDefinitionInitCommand(c common.Args) *cobra.Command {
 	cmd.Flags().StringP(FlagTemplateYAML, "f", "", "Specify the template yaml file that definition will use to build the schema. If empty, a default template for the given definition type will be used.")
 	cmd.Flags().StringP(FlagOutput, "o", "", "Specify the output path of the generated definition. If empty, the definition will be printed in the console.")
 	cmd.Flags().BoolP(FlagInteractive, "i", false, "Specify whether use interactive process to help generate definitions.")
-	cmd.Flags().StringP(FlagProvider, "p", "", "Specify which provider the cloud resource definition belongs to. Only `alibaba`, `aws`, `azure` are supported.")
+	cmd.Flags().StringP(FlagProvider, "p", "", "Specify which provider the cloud resource definition belongs to. Only `alibaba`, `aws`, `azure`, `gcp`, `baidu`, `tencent`, `elastic`, `ucloud`, `vsphere` are supported.")
 	cmd.Flags().StringP(FlagGit, "", "", "Specify which git repository the configuration(HCL) is stored in. Valid when --provider/-p is set.")
 	cmd.Flags().StringP(FlagLocal, "", "", "Specify the local path of the configuration(HCL) file. Valid when --provider/-p is set.")
 	cmd.Flags().StringP(FlagPath, "", "", "Specify which path the configuration(HCL) is stored in the Git repository. Valid when --git is set.")
@@ -302,7 +302,7 @@ func generateTerraformTypedComponentDefinition(cmd *cobra.Command, name, kind, p
 	}
 
 	switch provider {
-	case "aws", "azure", "alibaba", "tencent", "gcp", "baidu", "elastic", "ucloud":
+	case "aws", "azure", "alibaba", "tencent", "gcp", "baidu", "elastic", "ucloud", "vsphere":
 		var terraform *commontype.Terraform
 
 		git, err := cmd.Flags().GetString(FlagGit)
@@ -378,7 +378,7 @@ func generateTerraformTypedComponentDefinition(cmd *cobra.Command, name, kind, p
 		}
 		return out.String(), nil
 	default:
-		return "", errors.Errorf("Provider `%s` is not supported. Only `alibaba`, `aws`, `azure`, `gcp`, `baidu`, `tencent`, `elastic`, `ucloud` are supported.", provider)
+		return "", errors.Errorf("Provider `%s` is not supported. Only `alibaba`, `aws`, `azure`, `gcp`, `baidu`, `tencent`, `elastic`, `ucloud`, `vsphere` are supported.", provider)
 	}
 }
 

references/cli/kube.go

@@ -213,7 +213,7 @@ var (
 		Apply Kubernetes objects in clusters
 
 		Apply Kubernetes objects in multiple clusters. Use --clusters to specify which clusters to
-		apply. If -n/--namespace is used, the original object namespace will be overrode.
+		apply. If -n/--namespace is used, the original object namespace will be overridden.
 
 		You can use -f/--file to specify the object file/folder to apply. Multiple file inputs are allowed.
 		Directory input and web url input is supported as well.

references/cli/up.go

@@ -268,7 +268,7 @@ var (
 		vela up example-app -n example-ns --publish-version beta --revision example-app-v2
 
 		# Deploy an application from stdin
-		cat <<EOF | vela up vela up -f -
+		cat <<EOF | vela up -f -
         ... <app.yaml here> ...
         EOF
 `))

test/e2e-multicluster-test/multicluster_standalone_test.go

@@ -43,7 +43,11 @@ import (
 )
 
 var _ = Describe("Test multicluster standalone scenario", func() {
-
+	waitObject := func(ctx context.Context, un unstructured.Unstructured) {
+		Eventually(func(g Gomega) error {
+			return k8sClient.Get(ctx, client.ObjectKeyFromObject(&un), &un)
+		}, 10*time.Second).Should(Succeed())
+	}
 	var namespace string
 	var hubCtx context.Context
 	var workerCtx context.Context
@@ -116,10 +120,13 @@ var _ = Describe("Test multicluster standalone scenario", func() {
 
 		deploy := readFile("deployment.yaml")
 		Expect(k8sClient.Create(hubCtx, deploy)).Should(Succeed())
+		waitObject(hubCtx, *deploy)
 		workflow := readFile("workflow-suspend.yaml")
 		Expect(k8sClient.Create(hubCtx, workflow)).Should(Succeed())
+		waitObject(hubCtx, *workflow)
 		policy := readFile("policy-zero-replica.yaml")
 		Expect(k8sClient.Create(hubCtx, policy)).Should(Succeed())
+		waitObject(hubCtx, *policy)
 		app := readFile("app-with-publish-version.yaml")
 		Expect(k8sClient.Create(hubCtx, app)).Should(Succeed())
 		appKey := client.ObjectKeyFromObject(app)

test/e2e-multicluster-test/multicluster_test.go

@@ -659,5 +659,23 @@ var _ = Describe("Test multicluster scenario", func() {
 				g.Expect(k8sClient.Get(hubCtx, types.NamespacedName{Namespace: testNamespace, Name: "another"}, &corev1.ConfigMap{})).Should(Satisfy(kerrors.IsNotFound))
 			})
 		})
+
+		It("Test Application with env in webservice and labels & storage trait", func() {
+			bs, err := ioutil.ReadFile("./testdata/app/app-with-env-labels-storage.yaml")
+			Expect(err).Should(Succeed())
+			app := &v1beta1.Application{}
+			Expect(yaml.Unmarshal(bs, app)).Should(Succeed())
+			app.SetNamespace(namespace)
+			Expect(k8sClient.Create(hubCtx, app)).Should(Succeed())
+			deploy := &appsv1.Deployment{}
+			Eventually(func(g Gomega) {
+				g.Expect(k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: "test"}, deploy)).Should(Succeed())
+			}, 15*time.Second).Should(Succeed())
+			Expect(deploy.GetLabels()["key"]).Should(Equal("val"))
+			Expect(len(deploy.Spec.Template.Spec.Containers[0].Env)).Should(Equal(1))
+			Expect(deploy.Spec.Template.Spec.Containers[0].Env[0].Name).Should(Equal("testKey"))
+			Expect(deploy.Spec.Template.Spec.Containers[0].Env[0].Value).Should(Equal("testValue"))
+			Expect(len(deploy.Spec.Template.Spec.Volumes)).Should(Equal(1))
+		})
 	})
 })

test/e2e-multicluster-test/suite_test.go

@@ -23,11 +23,15 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/strings/slices"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
+	"github.com/oam-dev/kubevela/apis/types"
 	"github.com/oam-dev/kubevela/pkg/multicluster"
+	oamutil "github.com/oam-dev/kubevela/pkg/oam/util"
 	"github.com/oam-dev/kubevela/pkg/utils/common"
 	"github.com/oam-dev/kubevela/pkg/utils/util"
 	"github.com/oam-dev/kubevela/references/cli"
@@ -73,12 +77,25 @@ var _ = BeforeSuite(func() {
 })
 
 var _ = AfterSuite(func() {
+	holdAddons := []string{"addon-terraform", "addon-fluxcd"}
 	Eventually(func(g Gomega) {
 		apps := &v1beta1.ApplicationList{}
 		g.Expect(k8sClient.List(context.Background(), apps)).Should(Succeed())
 		for _, app := range apps.Items {
+			if slices.Contains(holdAddons, app.Name) {
+				continue
+			}
 			g.Expect(k8sClient.Delete(context.Background(), app.DeepCopy())).Should(Succeed())
 		}
+	}, 3*time.Minute).Should(Succeed())
+	Eventually(func(g Gomega) {
+		// Delete terraform and fluxcd in order
+		app := &v1beta1.Application{}
+		apps := &v1beta1.ApplicationList{}
+		for _, addon := range holdAddons {
+			g.Expect(k8sClient.Delete(context.Background(), &v1beta1.Application{ObjectMeta: v1.ObjectMeta{Name: addon, Namespace: types.DefaultKubeVelaNS}})).Should(SatisfyAny(Succeed(), oamutil.NotFoundMatcher{}))
+			g.Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: addon, Namespace: types.DefaultKubeVelaNS}, app)).Should(SatisfyAny(Succeed(), oamutil.NotFoundMatcher{}))
+		}
 		err := k8sClient.List(context.Background(), apps)
 		g.Expect(err, nil)
 		g.Expect(len(apps.Items)).Should(Equal(0))

test/e2e-multicluster-test/testdata/app/app-bad-resource.yaml

@@ -8,7 +8,7 @@ spec:
     - name: bad-resource
       properties:
         objects:
-        - apiVersion: apiregistration.k8s.io/v1beta1
+        - apiVersion: apiregistration.k8s.io/v1
           kind: APIService
           metadata:
             name: test-bad-resource

test/e2e-multicluster-test/testdata/app/app-with-env-labels-storage.yaml

@@ -0,0 +1,35 @@
+apiVersion: core.oam.dev/v1beta1
+kind: Application
+metadata:
+  name: test
+spec:
+  components:
+    - name: test
+      properties:
+        cmd:
+          - sleep
+          - -c
+          - "86400"
+        env:
+          - name: testKey
+            value: testValue
+        image: busybox
+        imagePullPolicy: IfNotPresent
+      traits:
+        - properties:
+            key: val
+          type: labels
+        - properties:
+            pvc:
+              - accessModes:
+                  - ReadWriteOnce
+                mountOnly: false
+                mountPath: /data
+                name: lvhyca
+                resources:
+                  requests:
+                    storage: 4096Mi
+                storageClassName: default
+                volumeMode: Filesystem
+          type: storage
+      type: webservice

test/e2e-test/health_scope_test.go

@@ -18,6 +18,7 @@ package controllers_test
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
@@ -27,6 +28,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
@@ -80,6 +82,7 @@ var _ = Describe("HealthScope", func() {
 		var healthyAppName, unhealthyAppName string
 		Expect(utilcommon.ReadYamlToObject("testdata/app/app_healthscope.yaml", &newApp)).Should(BeNil())
 		newApp.Namespace = namespace
+		convertToLegacyIngressTrait(&newApp)
 		Eventually(func() error {
 			return k8sClient.Create(ctx, newApp.DeepCopy())
 		}, 10*time.Second, 500*time.Millisecond).Should(Succeed())
@@ -101,6 +104,7 @@ var _ = Describe("HealthScope", func() {
 		newApp = v1beta1.Application{}
 		Expect(utilcommon.ReadYamlToObject("testdata/app/app_healthscope_unhealthy.yaml", &newApp)).Should(BeNil())
 		newApp.Namespace = namespace
+		convertToLegacyIngressTrait(&newApp)
 		Eventually(func() error {
 			return k8sClient.Create(ctx, newApp.DeepCopy())
 		}, 10*time.Second, 500*time.Millisecond).Should(Succeed())
@@ -154,9 +158,11 @@ var _ = Describe("HealthScope", func() {
 				return fmt.Errorf("expect healthy comp, but %v is unhealthy, msg: %q", compSts1.Name, compSts1.Message)
 			}
 			if len(compSts1.Traits) != 1 {
-				return fmt.Errorf("expect 2 traits statuses, but got %d", len(compSts1.Traits))
+				return fmt.Errorf("expect 1 trait statuses, but got %d", len(compSts1.Traits))
+			}
+			if !strings.Contains(compSts1.Traits[0].Message, "visit the cluster or load balancer in front of the cluster") {
+				return fmt.Errorf("trait message isn't right, now is %s", compSts1.Traits[0].Message)
 			}
-			Expect(compSts1.Traits[0].Message).Should(ContainSubstring("No loadBalancer found"))
 
 			return nil
 		}, time.Second*30, time.Millisecond*500).Should(Succeed())
@@ -205,3 +211,34 @@ var _ = Describe("HealthScope", func() {
 		}, time.Second*30, time.Millisecond*500).Should(Succeed())
 	})
 })
+
+// convertToLegacyIngressTrait convert app's gateway trait to ingress
+func convertToLegacyIngressTrait(app *v1beta1.Application) {
+	if noNetworkingV1 {
+		for i := range app.Spec.Components {
+			for j := range app.Spec.Components[i].Traits {
+				if app.Spec.Components[i].Traits[j].Type == "gateway" {
+					app.Spec.Components[i].Traits[j].Type = "ingress"
+				}
+				props := app.Spec.Components[i].Traits[j].Properties
+				propMap, err := util.RawExtension2Map(props)
+				if err != nil {
+					return
+				}
+				newPropMap := map[string]interface{}{}
+				for k := range propMap {
+					if k != "class" {
+						newPropMap[k] = propMap[k]
+					}
+				}
+				ext, err := json.Marshal(newPropMap)
+				if err != nil {
+					return
+				}
+				app.Spec.Components[i].Traits[j].Properties = &runtime.RawExtension{
+					Raw: ext,
+				}
+			}
+		}
+	}
+}

test/e2e-test/suite_test.go

@@ -31,13 +31,17 @@ import (
 	. "github.com/onsi/gomega"
 
 	kruise "github.com/openkruise/kruise-api/apps/v1alpha1"
+	"github.com/pkg/errors"
+	networkv1 "k8s.io/api/networking/v1"
 	rbac "k8s.io/api/rbac/v1"
 	crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	k8sutils "k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
@@ -58,6 +62,10 @@ var manualscalertrait v1alpha2.TraitDefinition
 var roleName = "oam-example-com"
 var roleBindingName = "oam-role-binding"
 
+var (
+	noNetworkingV1 bool
+)
+
 // A DefinitionExtension is an Object type for xxxDefinitin.spec.extension
 type DefinitionExtension struct {
 	Alias string `json:"alias,omitempty"`
@@ -104,6 +112,8 @@ var _ = BeforeSuite(func(done Done) {
 	}
 	By("Finished setting up test environment")
 
+	detectAPIVersion()
+
 	// Create manual scaler trait definition
 	manualscalertrait = v1alpha2.TraitDefinition{
 		ObjectMeta: metav1.ObjectMeta{
@@ -219,3 +229,20 @@ func RequestReconcileNow(ctx context.Context, o client.Object) {
 func randomNamespaceName(basic string) string {
 	return fmt.Sprintf("%s-%s", basic, strconv.FormatInt(rand.Int63(), 16))
 }
+
+// detectAPIVersion helps detect legacy GVK
+func detectAPIVersion() {
+	err := k8sClient.Create(context.Background(), &networkv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-ingress",
+		},
+		Spec: networkv1.IngressSpec{
+			IngressClassName: k8sutils.StringPtr("nginx"),
+			Rules:            []networkv1.IngressRule{},
+		},
+	})
+	var noKindMatchErr = &meta.NoKindMatchError{}
+	if err != nil && errors.As(err, &noKindMatchErr) {
+		noNetworkingV1 = true
+	}
+}

test/e2e-test/testdata/app/app_healthscope.yaml

@@ -13,8 +13,9 @@ spec:
         image: oamdev/testapp:v1
         port: 8080
       traits:
-        - type: ingress
+        - type: gateway
           properties:
+            class: traefik
             domain: test.my.domain
             http:
               "/": 8080
@@ -27,8 +28,9 @@ spec:
         image: oamdev/testapp:v1
         port: 8080
       traits:
-        - type: ingress
+        - type: gateway
           properties:
+            class: traefik
             domain: test.my.domain
             http:
               "/": 8080

test/e2e-test/testdata/app/app_healthscope_unhealthy.yaml

@@ -21,8 +21,9 @@ spec:
         image: oamdev/testapp:v1
         port: 8080
       traits:
-        - type: ingress
+        - type: gateway
           properties:
+            class: traefik
             domain: test.my.domain
             http:
               "/": 8080

vela-templates/definitions/internal/policy/topology.cue

@@ -9,7 +9,7 @@
 template: {
 	parameter: {
 		// +usage=Specify the names of the clusters to select.
-		cluster?: [...string]
+		clusters?: [...string]
 		// +usage=Specify the label selector for clusters
 		clusterLabelSelector?: [string]: string
 		// +usage=Deprecated: Use clusterLabelSelector instead.