mirror of
https://github.com/kubevela/kubevela.git
synced 2026-08-18 03:56:36 +00:00
Test: add e2e-test for multi-cluster rollout (#2472)
* Test: add e2e-test guarantee multi-cluster rollout * Fix: install rollout runtime chart add cluster name in dockerfile Fix: build image error enable all test Fix: add rollout chart add shebang fix image repo
This commit is contained in:
wyike
@@ -76,10 +76,16 @@ jobs:
|
||||
- name: Load Image to kind cluster (Hub)
|
||||
run: make kind-load
|
||||
|
||||
- name: Load Image to kind cluster (Worker)
|
||||
run: |
|
||||
make kind-load-runtime-cluster
|
||||
|
||||
- name: Cleanup for e2e tests
|
||||
run: |
|
||||
make e2e-cleanup
|
||||
make e2e-setup-core
|
||||
make
|
||||
make setup-runtime-e2e-cluster
|
||||
|
||||
- name: Run e2e multicluster tests
|
||||
run: make e2e-multicluster-test
|
||||
|
||||
+4
-1
@@ -45,4 +45,7 @@ charts/vela-core/crds/_.yaml
|
||||
.vela/
|
||||
|
||||
# check docs
|
||||
git-page/
|
||||
git-page/
|
||||
|
||||
# e2e rollout runtime image build
|
||||
runtime/rollout/e2e/tmp
|
||||
@@ -39,6 +39,9 @@ endif
|
||||
VELA_CORE_IMAGE ?= vela-core:latest
|
||||
VELA_CORE_TEST_IMAGE ?= vela-core-test:$(GIT_COMMIT)
|
||||
VELA_RUNTIME_ROLLOUT_IMAGE ?= vela-runtime-rollout:latest
|
||||
VELA_RUNTIME_ROLLOUT_TEST_IMAGE ?= vela-runtime-rollout-test:$(GIT_COMMIT)
|
||||
RUNTIME_CLUSTER_CONFIG ?= /tmp/worker.kubeconfig
|
||||
RUNTIME_CLUSTER_NAME ?= worker
|
||||
|
||||
all: build
|
||||
|
||||
@@ -146,6 +149,9 @@ e2e-setup-core:
|
||||
helm upgrade --install --create-namespace --namespace vela-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set applicationRevisionLimit=5 --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --set multicluster.enabled=true --wait kubevela ./charts/vela-core
|
||||
kubectl wait --for=condition=Available deployment/kubevela-vela-core -n vela-system --timeout=180s
|
||||
|
||||
setup-runtime-e2e-cluster:
|
||||
helm upgrade --install --create-namespace --namespace vela-system --kubeconfig=$(RUNTIME_CLUSTER_CONFIG) --set image.pullPolicy=IfNotPresent --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) --wait vela-rollout ./runtime/rollout/charts
|
||||
|
||||
e2e-setup:
|
||||
helm install kruise https://github.com/openkruise/kruise/releases/download/v0.9.0/kruise-chart.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true"
|
||||
sh ./hack/e2e/modify_charts.sh
|
||||
@@ -216,6 +222,12 @@ kind-load:
|
||||
docker build -t $(VELA_CORE_TEST_IMAGE) -f Dockerfile.e2e .
|
||||
kind load docker-image $(VELA_CORE_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_CORE_TEST_IMAGE)"; exit 1; }
|
||||
|
||||
kind-load-runtime-cluster:
|
||||
/bin/sh hack/e2e/build_runtime_rollout.sh
|
||||
docker build -t $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) -f runtime/rollout/e2e/Dockerfile.e2e runtime/rollout/e2e/
|
||||
rm -rf runtime/rollout/e2e/tmp
|
||||
kind load docker-image $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) --name=$(RUNTIME_CLUSTER_NAME) || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; }
|
||||
|
||||
# Run tests
|
||||
core-test: fmt vet manifests
|
||||
go test ./pkg/... -coverprofile cover.out
|
||||
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
|
||||
TEMP_DIR="./runtime/rollout/e2e/tmp/"
|
||||
|
||||
mkdir -p $TEMP_DIR
|
||||
cp -r go.mod $TEMP_DIR
|
||||
cp -r go.sum $TEMP_DIR
|
||||
cp -r entrypoint.sh $TEMP_DIR
|
||||
cp -r runtime/rollout/cmd/main.go $TEMP_DIR
|
||||
cp -r ./apis $TEMP_DIR
|
||||
cp -r ./pkg $TEMP_DIR
|
||||
cp -r ./version $TEMP_DIR
|
||||
@@ -0,0 +1,44 @@
|
||||
# Build the manager binary
|
||||
FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.16-alpine as builder
|
||||
|
||||
WORKDIR /workspace
|
||||
# Copy the Go Modules manifests
|
||||
COPY ./tmp/go.mod go.mod
|
||||
COPY ./tmp/go.sum go.sum
|
||||
# cache deps before building and copying source so that we don't need to re-download as much
|
||||
# and so that source changes don't invalidate our downloaded layer
|
||||
RUN go mod download
|
||||
|
||||
# Copy the go source
|
||||
COPY ./tmp/main.go main.go
|
||||
COPY ./tmp/apis apis/
|
||||
COPY ./tmp/pkg pkg/
|
||||
COPY ./tmp/version version/
|
||||
|
||||
# Build
|
||||
ARG TARGETARCH
|
||||
ARG VERSION
|
||||
ARG GITVERSION
|
||||
RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \
|
||||
go build -a -ldflags "-s -w -X github.com/oam-dev/kubevela/version.VelaVersion=${VERSION:-undefined} -X github.com/oam-dev/kubevela/version.GitRevision=${GITVERSION:-undefined}" \
|
||||
-o manager-${TARGETARCH} main.go
|
||||
|
||||
# Use alpine as base image due to the discussion in issue #1448
|
||||
# You can replace distroless as minimal base image to package the manager binary
|
||||
# Refer to https://github.com/GoogleContainerTools/distroless for more details
|
||||
# Overwrite `BASE_IMAGE` by passing `--build-arg=BASE_IMAGE=gcr.io/distroless/static:nonroot`
|
||||
ARG BASE_IMAGE
|
||||
FROM ${BASE_IMAGE:-alpine:latest}
|
||||
# This is required by daemon connnecting with cri
|
||||
RUN apk add --no-cache ca-certificates bash
|
||||
|
||||
WORKDIR /
|
||||
|
||||
ARG TARGETARCH
|
||||
COPY --from=builder /workspace/manager-${TARGETARCH} /usr/local/bin/manager
|
||||
|
||||
COPY ./tmp/entrypoint.sh /usr/local/bin/
|
||||
|
||||
ENTRYPOINT ["entrypoint.sh"]
|
||||
|
||||
CMD ["manager"]
|
||||
@@ -0,0 +1,147 @@
|
||||
/*
|
||||
Copyright 2021 The KubeVela Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e_multicluster_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
|
||||
"github.com/oam-dev/kubevela/apis/standard.oam.dev/v1alpha1"
|
||||
"github.com/oam-dev/kubevela/pkg/oam/util"
|
||||
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
var _ = Describe("Test MultiClustet Rollout", func() {
|
||||
Context("Test Runtime Cluster Rollout", func() {
|
||||
var namespace string
|
||||
var hubCtx context.Context
|
||||
var workerCtx context.Context
|
||||
var rollout v1alpha1.Rollout
|
||||
var componentName string
|
||||
var targetDeploy appsv1.Deployment
|
||||
var sourceDeploy appsv1.Deployment
|
||||
|
||||
BeforeEach(func() {
|
||||
hubCtx, workerCtx, namespace = initializeContextAndNamespace()
|
||||
componentName = "hello-world-server"
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
cleanUpNamespace(hubCtx, workerCtx, namespace)
|
||||
ns := v1.Namespace{}
|
||||
Eventually(func() error { return k8sClient.Get(hubCtx, types.NamespacedName{Name: namespace}, &ns) }, 300*time.Second, 300*time.Millisecond).Should(util.NotFoundMatcher{})
|
||||
})
|
||||
|
||||
verifySucceed := func(componentRevision string) {
|
||||
By("check rollout status have succeed")
|
||||
Eventually(func() error {
|
||||
rolloutKey := types.NamespacedName{Namespace: namespace, Name: componentName}
|
||||
if err := k8sClient.Get(workerCtx, rolloutKey, &rollout); err != nil {
|
||||
return err
|
||||
}
|
||||
if rollout.Spec.TargetRevisionName != componentRevision {
|
||||
return fmt.Errorf("rollout have not point to right targetRevision")
|
||||
}
|
||||
if rollout.Status.RollingState != v1alpha1.RolloutSucceedState {
|
||||
return fmt.Errorf("error rollout status state %s", rollout.Status.RollingState)
|
||||
}
|
||||
compRevName := rollout.Spec.TargetRevisionName
|
||||
deployKey := types.NamespacedName{Namespace: namespace, Name: compRevName}
|
||||
if err := k8sClient.Get(workerCtx, deployKey, &targetDeploy); err != nil {
|
||||
return err
|
||||
}
|
||||
if *targetDeploy.Spec.Replicas != *rollout.Spec.RolloutPlan.TargetSize {
|
||||
return fmt.Errorf("targetDeploy replicas missMatch %d, %d", targetDeploy.Spec.Replicas, rollout.Spec.RolloutPlan.TargetSize)
|
||||
}
|
||||
if targetDeploy.Status.UpdatedReplicas != *targetDeploy.Spec.Replicas {
|
||||
return fmt.Errorf("update not finish")
|
||||
}
|
||||
if len(targetDeploy.OwnerReferences) != 1 {
|
||||
return fmt.Errorf("workload ownerReference missMatch")
|
||||
}
|
||||
// guarantee rollout's owners and workload's owners are same
|
||||
if targetDeploy.OwnerReferences[0].Kind != rollout.OwnerReferences[0].Kind ||
|
||||
targetDeploy.OwnerReferences[0].Name != rollout.OwnerReferences[0].Name {
|
||||
return fmt.Errorf("workload ownerReference missMatch")
|
||||
}
|
||||
if rollout.Status.LastSourceRevision == "" {
|
||||
return nil
|
||||
}
|
||||
deployKey = types.NamespacedName{Namespace: namespace, Name: rollout.Status.LastSourceRevision}
|
||||
if err := k8sClient.Get(workerCtx, deployKey, &sourceDeploy); err == nil || !apierrors.IsNotFound(err) {
|
||||
return fmt.Errorf("source deploy still exist")
|
||||
}
|
||||
return nil
|
||||
}, time.Second*360, 300*time.Millisecond).Should(BeNil())
|
||||
}
|
||||
|
||||
It("Test Rollout whole feature in runtime cluster ", func() {
|
||||
app := &v1beta1.Application{}
|
||||
appYaml, err := ioutil.ReadFile("./testdata/app/app-rollout-envbinding.yaml")
|
||||
Expect(err).Should(Succeed())
|
||||
Expect(yaml.Unmarshal([]byte(appYaml), app)).Should(Succeed())
|
||||
app.SetNamespace(namespace)
|
||||
err = k8sClient.Create(hubCtx, app)
|
||||
Expect(err).Should(Succeed())
|
||||
verifySucceed(componentName + "-v1")
|
||||
|
||||
By("update application to v2")
|
||||
checkApp := &v1beta1.Application{}
|
||||
Eventually(func() error {
|
||||
if err := k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp); err != nil {
|
||||
return err
|
||||
}
|
||||
checkApp.Spec.Components[0].Properties.Raw = []byte(`{"image": "stefanprodan/podinfo:5.0.2"}`)
|
||||
if err := k8sClient.Update(hubCtx, checkApp); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}, 500*time.Millisecond, 30*time.Second).Should(BeNil())
|
||||
verifySucceed(componentName + "-v2")
|
||||
|
||||
By("revert to v1, should guarantee compRev v1 still exist")
|
||||
appYaml, err = ioutil.ReadFile("./testdata/app/revert-app-envbinding.yaml")
|
||||
Expect(err).Should(Succeed())
|
||||
|
||||
Expect(k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp)).Should(BeNil())
|
||||
revertApp := &v1beta1.Application{}
|
||||
Expect(yaml.Unmarshal([]byte(appYaml), revertApp)).Should(Succeed())
|
||||
revertApp.SetNamespace(namespace)
|
||||
revertApp.SetResourceVersion(checkApp.ResourceVersion)
|
||||
|
||||
Eventually(func() error {
|
||||
if err := k8sClient.Update(hubCtx, revertApp); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}, 500*time.Millisecond, 30*time.Second).Should(BeNil())
|
||||
verifySucceed(componentName + "-v1")
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,37 @@
|
||||
apiVersion: core.oam.dev/v1beta1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: example-app
|
||||
namespace: default
|
||||
spec:
|
||||
components:
|
||||
- name: hello-world-server
|
||||
type: webservice
|
||||
properties:
|
||||
image: stefanprodan/podinfo:4.0.3
|
||||
traits:
|
||||
- type: rollout
|
||||
properties:
|
||||
targetSize: 2
|
||||
rolloutBatches:
|
||||
- replicas: 1
|
||||
- replicas: 1
|
||||
|
||||
policies:
|
||||
- name: example-multi-env-policy
|
||||
type: env-binding
|
||||
properties:
|
||||
envs:
|
||||
- name: staging
|
||||
placement: # selecting the cluster to deploy to
|
||||
clusterSelector:
|
||||
name: cluster-worker
|
||||
|
||||
workflow:
|
||||
steps:
|
||||
# deploy to staging env
|
||||
- name: deploy-staging
|
||||
type: deploy2env
|
||||
properties:
|
||||
policy: example-multi-env-policy
|
||||
env: staging
|
||||
@@ -0,0 +1,38 @@
|
||||
apiVersion: core.oam.dev/v1beta1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: example-app
|
||||
namespace: default
|
||||
spec:
|
||||
components:
|
||||
- name: hello-world-server
|
||||
type: webservice
|
||||
properties:
|
||||
image: stefanprodan/podinfo:5.0.2
|
||||
traits:
|
||||
- type: rollout
|
||||
properties:
|
||||
targetRevision: hello-world-server-v1
|
||||
targetSize: 2
|
||||
rolloutBatches:
|
||||
- replicas: 1
|
||||
- replicas: 1
|
||||
|
||||
policies:
|
||||
- name: example-multi-env-policy
|
||||
type: env-binding
|
||||
properties:
|
||||
envs:
|
||||
- name: staging
|
||||
placement: # selecting the cluster to deploy to
|
||||
clusterSelector:
|
||||
name: cluster-worker
|
||||
|
||||
workflow:
|
||||
steps:
|
||||
# deploy to staging env
|
||||
- name: deploy-staging
|
||||
type: deploy2env
|
||||
properties:
|
||||
policy: example-multi-env-policy
|
||||
env: staging
|
||||
Reference in New Issue
Block a user