diff --git a/.github/workflows/e2e-multicluster-test.yml b/.github/workflows/e2e-multicluster-test.yml index ee9a9b63b..f6852efc7 100644 --- a/.github/workflows/e2e-multicluster-test.yml +++ b/.github/workflows/e2e-multicluster-test.yml @@ -76,10 +76,16 @@ jobs: - name: Load Image to kind cluster (Hub) run: make kind-load + - name: Load Image to kind cluster (Worker) + run: | + make kind-load-runtime-cluster + - name: Cleanup for e2e tests run: | make e2e-cleanup make e2e-setup-core + make + make setup-runtime-e2e-cluster - name: Run e2e multicluster tests run: make e2e-multicluster-test diff --git a/.gitignore b/.gitignore index 9b04fd190..b014a27ff 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,7 @@ charts/vela-core/crds/_.yaml .vela/ # check docs -git-page/ \ No newline at end of file +git-page/ + +# e2e rollout runtime image build +runtime/rollout/e2e/tmp \ No newline at end of file diff --git a/Makefile b/Makefile index ed2641e44..de5dbf7cb 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,9 @@ endif VELA_CORE_IMAGE ?= vela-core:latest VELA_CORE_TEST_IMAGE ?= vela-core-test:$(GIT_COMMIT) VELA_RUNTIME_ROLLOUT_IMAGE ?= vela-runtime-rollout:latest +VELA_RUNTIME_ROLLOUT_TEST_IMAGE ?= vela-runtime-rollout-test:$(GIT_COMMIT) +RUNTIME_CLUSTER_CONFIG ?= /tmp/worker.kubeconfig +RUNTIME_CLUSTER_NAME ?= worker all: build @@ -146,6 +149,9 @@ e2e-setup-core: helm upgrade --install --create-namespace --namespace vela-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set applicationRevisionLimit=5 --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --set multicluster.enabled=true --wait kubevela ./charts/vela-core kubectl wait --for=condition=Available deployment/kubevela-vela-core -n vela-system --timeout=180s +setup-runtime-e2e-cluster: + helm upgrade --install --create-namespace --namespace vela-system --kubeconfig=$(RUNTIME_CLUSTER_CONFIG) --set image.pullPolicy=IfNotPresent --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) --wait vela-rollout ./runtime/rollout/charts + e2e-setup: helm install kruise https://github.com/openkruise/kruise/releases/download/v0.9.0/kruise-chart.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true" sh ./hack/e2e/modify_charts.sh @@ -216,6 +222,12 @@ kind-load: docker build -t $(VELA_CORE_TEST_IMAGE) -f Dockerfile.e2e . kind load docker-image $(VELA_CORE_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_CORE_TEST_IMAGE)"; exit 1; } +kind-load-runtime-cluster: + /bin/sh hack/e2e/build_runtime_rollout.sh + docker build -t $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) -f runtime/rollout/e2e/Dockerfile.e2e runtime/rollout/e2e/ + rm -rf runtime/rollout/e2e/tmp + kind load docker-image $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) --name=$(RUNTIME_CLUSTER_NAME) || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; } + # Run tests core-test: fmt vet manifests go test ./pkg/... -coverprofile cover.out diff --git a/hack/e2e/build_runtime_rollout.sh b/hack/e2e/build_runtime_rollout.sh new file mode 100755 index 000000000..6e76d6a2b --- /dev/null +++ b/hack/e2e/build_runtime_rollout.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +TEMP_DIR="./runtime/rollout/e2e/tmp/" + +mkdir -p $TEMP_DIR +cp -r go.mod $TEMP_DIR +cp -r go.sum $TEMP_DIR +cp -r entrypoint.sh $TEMP_DIR +cp -r runtime/rollout/cmd/main.go $TEMP_DIR +cp -r ./apis $TEMP_DIR +cp -r ./pkg $TEMP_DIR +cp -r ./version $TEMP_DIR diff --git a/runtime/rollout/e2e/Dockerfile.e2e b/runtime/rollout/e2e/Dockerfile.e2e new file mode 100644 index 000000000..82c13c5b8 --- /dev/null +++ b/runtime/rollout/e2e/Dockerfile.e2e @@ -0,0 +1,44 @@ +# Build the manager binary +FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.16-alpine as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY ./tmp/go.mod go.mod +COPY ./tmp/go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY ./tmp/main.go main.go +COPY ./tmp/apis apis/ +COPY ./tmp/pkg pkg/ +COPY ./tmp/version version/ + +# Build +ARG TARGETARCH +ARG VERSION +ARG GITVERSION +RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \ + go build -a -ldflags "-s -w -X github.com/oam-dev/kubevela/version.VelaVersion=${VERSION:-undefined} -X github.com/oam-dev/kubevela/version.GitRevision=${GITVERSION:-undefined}" \ + -o manager-${TARGETARCH} main.go + +# Use alpine as base image due to the discussion in issue #1448 +# You can replace distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +# Overwrite `BASE_IMAGE` by passing `--build-arg=BASE_IMAGE=gcr.io/distroless/static:nonroot` +ARG BASE_IMAGE +FROM ${BASE_IMAGE:-alpine:latest} +# This is required by daemon connnecting with cri +RUN apk add --no-cache ca-certificates bash + +WORKDIR / + +ARG TARGETARCH +COPY --from=builder /workspace/manager-${TARGETARCH} /usr/local/bin/manager + +COPY ./tmp/entrypoint.sh /usr/local/bin/ + +ENTRYPOINT ["entrypoint.sh"] + +CMD ["manager"] diff --git a/test/e2e-multicluster-test/multicluster_rollout_test.go b/test/e2e-multicluster-test/multicluster_rollout_test.go new file mode 100644 index 000000000..98bd7023f --- /dev/null +++ b/test/e2e-multicluster-test/multicluster_rollout_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_multicluster_test + +import ( + "context" + "fmt" + "io/ioutil" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/apis/standard.oam.dev/v1alpha1" + "github.com/oam-dev/kubevela/pkg/oam/util" + + "sigs.k8s.io/yaml" +) + +var _ = Describe("Test MultiClustet Rollout", func() { + Context("Test Runtime Cluster Rollout", func() { + var namespace string + var hubCtx context.Context + var workerCtx context.Context + var rollout v1alpha1.Rollout + var componentName string + var targetDeploy appsv1.Deployment + var sourceDeploy appsv1.Deployment + + BeforeEach(func() { + hubCtx, workerCtx, namespace = initializeContextAndNamespace() + componentName = "hello-world-server" + }) + + AfterEach(func() { + cleanUpNamespace(hubCtx, workerCtx, namespace) + ns := v1.Namespace{} + Eventually(func() error { return k8sClient.Get(hubCtx, types.NamespacedName{Name: namespace}, &ns) }, 300*time.Second, 300*time.Millisecond).Should(util.NotFoundMatcher{}) + }) + + verifySucceed := func(componentRevision string) { + By("check rollout status have succeed") + Eventually(func() error { + rolloutKey := types.NamespacedName{Namespace: namespace, Name: componentName} + if err := k8sClient.Get(workerCtx, rolloutKey, &rollout); err != nil { + return err + } + if rollout.Spec.TargetRevisionName != componentRevision { + return fmt.Errorf("rollout have not point to right targetRevision") + } + if rollout.Status.RollingState != v1alpha1.RolloutSucceedState { + return fmt.Errorf("error rollout status state %s", rollout.Status.RollingState) + } + compRevName := rollout.Spec.TargetRevisionName + deployKey := types.NamespacedName{Namespace: namespace, Name: compRevName} + if err := k8sClient.Get(workerCtx, deployKey, &targetDeploy); err != nil { + return err + } + if *targetDeploy.Spec.Replicas != *rollout.Spec.RolloutPlan.TargetSize { + return fmt.Errorf("targetDeploy replicas missMatch %d, %d", targetDeploy.Spec.Replicas, rollout.Spec.RolloutPlan.TargetSize) + } + if targetDeploy.Status.UpdatedReplicas != *targetDeploy.Spec.Replicas { + return fmt.Errorf("update not finish") + } + if len(targetDeploy.OwnerReferences) != 1 { + return fmt.Errorf("workload ownerReference missMatch") + } + // guarantee rollout's owners and workload's owners are same + if targetDeploy.OwnerReferences[0].Kind != rollout.OwnerReferences[0].Kind || + targetDeploy.OwnerReferences[0].Name != rollout.OwnerReferences[0].Name { + return fmt.Errorf("workload ownerReference missMatch") + } + if rollout.Status.LastSourceRevision == "" { + return nil + } + deployKey = types.NamespacedName{Namespace: namespace, Name: rollout.Status.LastSourceRevision} + if err := k8sClient.Get(workerCtx, deployKey, &sourceDeploy); err == nil || !apierrors.IsNotFound(err) { + return fmt.Errorf("source deploy still exist") + } + return nil + }, time.Second*360, 300*time.Millisecond).Should(BeNil()) + } + + It("Test Rollout whole feature in runtime cluster ", func() { + app := &v1beta1.Application{} + appYaml, err := ioutil.ReadFile("./testdata/app/app-rollout-envbinding.yaml") + Expect(err).Should(Succeed()) + Expect(yaml.Unmarshal([]byte(appYaml), app)).Should(Succeed()) + app.SetNamespace(namespace) + err = k8sClient.Create(hubCtx, app) + Expect(err).Should(Succeed()) + verifySucceed(componentName + "-v1") + + By("update application to v2") + checkApp := &v1beta1.Application{} + Eventually(func() error { + if err := k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp); err != nil { + return err + } + checkApp.Spec.Components[0].Properties.Raw = []byte(`{"image": "stefanprodan/podinfo:5.0.2"}`) + if err := k8sClient.Update(hubCtx, checkApp); err != nil { + return err + } + return nil + }, 500*time.Millisecond, 30*time.Second).Should(BeNil()) + verifySucceed(componentName + "-v2") + + By("revert to v1, should guarantee compRev v1 still exist") + appYaml, err = ioutil.ReadFile("./testdata/app/revert-app-envbinding.yaml") + Expect(err).Should(Succeed()) + + Expect(k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp)).Should(BeNil()) + revertApp := &v1beta1.Application{} + Expect(yaml.Unmarshal([]byte(appYaml), revertApp)).Should(Succeed()) + revertApp.SetNamespace(namespace) + revertApp.SetResourceVersion(checkApp.ResourceVersion) + + Eventually(func() error { + if err := k8sClient.Update(hubCtx, revertApp); err != nil { + return err + } + return nil + }, 500*time.Millisecond, 30*time.Second).Should(BeNil()) + verifySucceed(componentName + "-v1") + }) + }) +}) diff --git a/test/e2e-multicluster-test/testdata/app/app-rollout-envbinding.yaml b/test/e2e-multicluster-test/testdata/app/app-rollout-envbinding.yaml new file mode 100644 index 000000000..4cb5a6765 --- /dev/null +++ b/test/e2e-multicluster-test/testdata/app/app-rollout-envbinding.yaml @@ -0,0 +1,37 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: example-app + namespace: default +spec: + components: + - name: hello-world-server + type: webservice + properties: + image: stefanprodan/podinfo:4.0.3 + traits: + - type: rollout + properties: + targetSize: 2 + rolloutBatches: + - replicas: 1 + - replicas: 1 + + policies: + - name: example-multi-env-policy + type: env-binding + properties: + envs: + - name: staging + placement: # selecting the cluster to deploy to + clusterSelector: + name: cluster-worker + + workflow: + steps: + # deploy to staging env + - name: deploy-staging + type: deploy2env + properties: + policy: example-multi-env-policy + env: staging diff --git a/test/e2e-multicluster-test/testdata/app/revert-app-envbinding.yaml b/test/e2e-multicluster-test/testdata/app/revert-app-envbinding.yaml new file mode 100644 index 000000000..d64b0a128 --- /dev/null +++ b/test/e2e-multicluster-test/testdata/app/revert-app-envbinding.yaml @@ -0,0 +1,38 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: example-app + namespace: default +spec: + components: + - name: hello-world-server + type: webservice + properties: + image: stefanprodan/podinfo:5.0.2 + traits: + - type: rollout + properties: + targetRevision: hello-world-server-v1 + targetSize: 2 + rolloutBatches: + - replicas: 1 + - replicas: 1 + + policies: + - name: example-multi-env-policy + type: env-binding + properties: + envs: + - name: staging + placement: # selecting the cluster to deploy to + clusterSelector: + name: cluster-worker + + workflow: + steps: + # deploy to staging env + - name: deploy-staging + type: deploy2env + properties: + policy: example-multi-env-policy + env: staging