From 237c71d94efc7a73bebdb9599d9571cab1462239 Mon Sep 17 00:00:00 2001 From: wyike Date: Thu, 28 Oct 2021 20:49:27 +0800 Subject: [PATCH] Backport 2527 to release 1.1 (#2555) * Fix: resolve confict * Fix: cherry pick 2472 to 1.1 --- .github/workflows/e2e-multicluster-test.yml | 6 + .gitignore | 5 +- Makefile | 12 + hack/e2e/build_runtime_rollout.sh | 12 + .../application/application_controller.go | 3 +- .../v1alpha2/application/apply.go | 21 -- .../v1alpha2/application/apply_test.go | 36 --- .../v1alpha2/application/assemble/assemble.go | 25 ++ .../assemble/assemble_suite_test.go | 47 ++++ .../core/scopes/healthscope/healthscope.go | 3 +- .../healthscope/healthscope_controller.go | 35 ++- .../v1alpha1/rollout/rollout_controller.go | 17 ++ pkg/oam/labels.go | 6 + runtime/rollout/cmd/main.go | 2 +- runtime/rollout/e2e/Dockerfile.e2e | 44 ++++ .../multicluster_rollout_test.go | 213 ++++++++++++++++++ .../testdata/app/app-rollout-envbinding.yaml | 37 +++ .../app/multi-cluster-health-policy.yaml | 48 ++++ .../testdata/app/revert-app-envbinding.yaml | 38 ++++ test/e2e-test/rollout_trait_test.go | 15 ++ 20 files changed, 558 insertions(+), 67 deletions(-) create mode 100755 hack/e2e/build_runtime_rollout.sh create mode 100644 runtime/rollout/e2e/Dockerfile.e2e create mode 100644 test/e2e-multicluster-test/multicluster_rollout_test.go create mode 100644 test/e2e-multicluster-test/testdata/app/app-rollout-envbinding.yaml create mode 100644 test/e2e-multicluster-test/testdata/app/multi-cluster-health-policy.yaml create mode 100644 test/e2e-multicluster-test/testdata/app/revert-app-envbinding.yaml diff --git a/.github/workflows/e2e-multicluster-test.yml b/.github/workflows/e2e-multicluster-test.yml index 6df12b4c3..942a50ea2 100644 --- a/.github/workflows/e2e-multicluster-test.yml +++ b/.github/workflows/e2e-multicluster-test.yml @@ -76,10 +76,16 @@ jobs: - name: Load Image to kind cluster (Hub) run: make kind-load + - name: Load Image to kind cluster (Worker) + run: | + make kind-load-runtime-cluster + - name: Cleanup for e2e tests run: | make e2e-cleanup make e2e-setup-core + make + make setup-runtime-e2e-cluster - name: Run e2e multicluster tests run: make e2e-multicluster-test diff --git a/.gitignore b/.gitignore index 9b04fd190..b014a27ff 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,7 @@ charts/vela-core/crds/_.yaml .vela/ # check docs -git-page/ \ No newline at end of file +git-page/ + +# e2e rollout runtime image build +runtime/rollout/e2e/tmp \ No newline at end of file diff --git a/Makefile b/Makefile index 599c67cd2..c0161a99f 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,9 @@ endif VELA_CORE_IMAGE ?= vela-core:latest VELA_CORE_TEST_IMAGE ?= vela-core-test:$(GIT_COMMIT) VELA_RUNTIME_ROLLOUT_IMAGE ?= vela-runtime-rollout:latest +VELA_RUNTIME_ROLLOUT_TEST_IMAGE ?= vela-runtime-rollout-test:$(GIT_COMMIT) +RUNTIME_CLUSTER_CONFIG ?= /tmp/worker.kubeconfig +RUNTIME_CLUSTER_NAME ?= worker all: build @@ -143,6 +146,9 @@ e2e-setup-core: helm upgrade --install --create-namespace --namespace vela-system --set image.pullPolicy=IfNotPresent --set image.repository=vela-core-test --set applicationRevisionLimit=5 --set dependCheckWait=10s --set image.tag=$(GIT_COMMIT) --set multicluster.enabled=true --wait kubevela ./charts/vela-core kubectl wait --for=condition=Available deployment/kubevela-vela-core -n vela-system --timeout=180s +setup-runtime-e2e-cluster: + helm upgrade --install --create-namespace --namespace vela-system --kubeconfig=$(RUNTIME_CLUSTER_CONFIG) --set image.pullPolicy=IfNotPresent --set image.repository=vela-runtime-rollout-test --set image.tag=$(GIT_COMMIT) --wait vela-rollout ./runtime/rollout/charts + e2e-setup: helm install kruise https://github.com/openkruise/kruise/releases/download/v0.9.0/kruise-chart.tgz --set featureGates="PreDownloadImageForInPlaceUpdate=true" sh ./hack/e2e/modify_charts.sh @@ -209,6 +215,12 @@ kind-load: docker build -t $(VELA_CORE_TEST_IMAGE) -f Dockerfile.e2e . kind load docker-image $(VELA_CORE_TEST_IMAGE) || { echo >&2 "kind not installed or error loading image: $(VELA_CORE_TEST_IMAGE)"; exit 1; } +kind-load-runtime-cluster: + /bin/sh hack/e2e/build_runtime_rollout.sh + docker build -t $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) -f runtime/rollout/e2e/Dockerfile.e2e runtime/rollout/e2e/ + rm -rf runtime/rollout/e2e/tmp + kind load docker-image $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE) --name=$(RUNTIME_CLUSTER_NAME) || { echo >&2 "kind not installed or error loading image: $(VELA_RUNTIME_ROLLOUT_TEST_IMAGE)"; exit 1; } + # Run tests core-test: fmt vet manifests go test ./pkg/... -coverprofile cover.out diff --git a/hack/e2e/build_runtime_rollout.sh b/hack/e2e/build_runtime_rollout.sh new file mode 100755 index 000000000..6e76d6a2b --- /dev/null +++ b/hack/e2e/build_runtime_rollout.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +TEMP_DIR="./runtime/rollout/e2e/tmp/" + +mkdir -p $TEMP_DIR +cp -r go.mod $TEMP_DIR +cp -r go.sum $TEMP_DIR +cp -r entrypoint.sh $TEMP_DIR +cp -r runtime/rollout/cmd/main.go $TEMP_DIR +cp -r ./apis $TEMP_DIR +cp -r ./pkg $TEMP_DIR +cp -r ./version $TEMP_DIR diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go index d9c1b1c83..c63729f27 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller.go @@ -43,6 +43,7 @@ import ( common2 "github.com/oam-dev/kubevela/pkg/controller/common" core "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev" "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha1/envbinding" + "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha2/application/assemble" "github.com/oam-dev/kubevela/pkg/cue/packages" "github.com/oam-dev/kubevela/pkg/oam" "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" @@ -223,7 +224,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return r.endWithNegativeCondition(ctx, app, condition.ErrorCondition("Render", err), common.ApplicationRendering) } - handler.handleCheckManageWorkloadTrait(handler.currentAppRev.Spec.TraitDefinitions, comps) + assemble.HandleCheckManageWorkloadTrait(*handler.currentAppRev, comps) if err := handler.HandleComponentsRevision(ctx, comps); err != nil { klog.ErrorS(err, "Failed to handle compoents revision", "application", klog.KObj(app)) diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/apply.go b/pkg/controller/core.oam.dev/v1alpha2/application/apply.go index 5ee75aacd..d933f4caf 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/apply.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/apply.go @@ -37,7 +37,6 @@ import ( "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev/v1alpha2/applicationrollout" "github.com/oam-dev/kubevela/pkg/controller/utils" "github.com/oam-dev/kubevela/pkg/cue/process" - "github.com/oam-dev/kubevela/pkg/oam" oamutil "github.com/oam-dev/kubevela/pkg/oam/util" ) @@ -304,26 +303,6 @@ func (h *AppHandler) aggregateHealthStatus(appFile *appfile.Appfile) ([]common.A return appStatus, healthy, nil } -func (h *AppHandler) handleCheckManageWorkloadTrait(traitDefs map[string]v1beta1.TraitDefinition, comps []*types.ComponentManifest) { - manageWorkloadTrait := map[string]bool{} - for traitName, definition := range traitDefs { - if definition.Spec.ManageWorkload { - manageWorkloadTrait[traitName] = true - } - } - if len(manageWorkloadTrait) == 0 { - return - } - for _, comp := range comps { - for _, trait := range comp.Traits { - traitType := trait.GetLabels()[oam.TraitTypeLabel] - if manageWorkloadTrait[traitType] { - trait.SetLabels(oamutil.MergeMapOverrideWithDst(trait.GetLabels(), map[string]string{oam.LabelManageWorkloadTrait: "true"})) - } - } - } -} - func generateScopeReference(scopes []appfile.Scope) []corev1.ObjectReference { var references []corev1.ObjectReference for _, scope := range scopes { diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/apply_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/apply_test.go index 7d313cdc5..c572c99ed 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/apply_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/apply_test.go @@ -30,7 +30,6 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -41,7 +40,6 @@ import ( "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" velatypes "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/appfile" - "github.com/oam-dev/kubevela/pkg/oam" ) const workloadDefinition = ` @@ -217,37 +215,3 @@ var _ = Describe("Test statusAggregate", func() { Expect(err).Should(BeNil()) }) }) - -var _ = Describe("Test handleCheckManageWorkloadTrait func", func() { - It("Test every situation", func() { - traitDefs := map[string]v1beta1.TraitDefinition{ - "rollout": v1beta1.TraitDefinition{ - Spec: v1beta1.TraitDefinitionSpec{ - ManageWorkload: true, - }, - }, - "normal": v1beta1.TraitDefinition{ - Spec: v1beta1.TraitDefinitionSpec{}, - }, - } - rolloutTrait := &unstructured.Unstructured{} - rolloutTrait.SetLabels(map[string]string{oam.TraitTypeLabel: "rollout"}) - - normalTrait := &unstructured.Unstructured{} - normalTrait.SetLabels(map[string]string{oam.TraitTypeLabel: "normal"}) - comps := []*velatypes.ComponentManifest{ - { - Traits: []*unstructured.Unstructured{ - rolloutTrait, - normalTrait, - }, - }, - } - h := AppHandler{} - h.handleCheckManageWorkloadTrait(traitDefs, comps) - Expect(len(rolloutTrait.GetLabels())).Should(BeEquivalentTo(2)) - Expect(rolloutTrait.GetLabels()[oam.LabelManageWorkloadTrait]).Should(BeEquivalentTo("true")) - Expect(len(normalTrait.GetLabels())).Should(BeEquivalentTo(1)) - Expect(normalTrait.GetLabels()[oam.LabelManageWorkloadTrait]).Should(BeEquivalentTo("")) - }) -}) diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/assemble/assemble.go b/pkg/controller/core.oam.dev/v1alpha2/application/assemble/assemble.go index 8b3ef598e..f11f41d02 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/assemble/assemble.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/assemble/assemble.go @@ -232,6 +232,9 @@ func PrepareBeforeApply(comp *types.ComponentManifest, appRev *v1beta1.Applicati } assembledTraits := make([]*unstructured.Unstructured, len(comp.Traits)) + + HandleCheckManageWorkloadTrait(*appRev, []*types.ComponentManifest{comp}) + for i, trait := range comp.Traits { setTraitLabels(trait, additionalLabel) assembledTraits[i] = trait @@ -329,3 +332,25 @@ func setTraitLabels(trait *unstructured.Unstructured, additionalLabels map[strin // add more trait-specific labels here util.AddLabels(trait, additionalLabels) } + +// HandleCheckManageWorkloadTrait will checkout every trait whether a manage-workload trait, if yes set label and annotation in trait +func HandleCheckManageWorkloadTrait(appRev v1beta1.ApplicationRevision, comps []*types.ComponentManifest) { + traitDefs := appRev.Spec.TraitDefinitions + manageWorkloadTrait := map[string]bool{} + for traitName, definition := range traitDefs { + if definition.Spec.ManageWorkload { + manageWorkloadTrait[traitName] = true + } + } + if len(manageWorkloadTrait) == 0 { + return + } + for _, comp := range comps { + for _, trait := range comp.Traits { + traitType := trait.GetLabels()[oam.TraitTypeLabel] + if manageWorkloadTrait[traitType] { + trait.SetLabels(util.MergeMapOverrideWithDst(trait.GetLabels(), map[string]string{oam.LabelManageWorkloadTrait: "true"})) + } + } + } +} diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/assemble/assemble_suite_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/assemble/assemble_suite_test.go index b8d1687f2..29a30a5e9 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/assemble/assemble_suite_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/assemble/assemble_suite_test.go @@ -26,6 +26,7 @@ import ( "sigs.k8s.io/yaml" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + velatypes "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/oam" ) @@ -203,3 +204,49 @@ var _ = Describe("Test Assemble Options", func() { Expect(wl.GetName()).Should(Equal(workloadName)) }) }) + +var _ = Describe("Test handleCheckManageWorkloadTrait func", func() { + It("Test every situation", func() { + traitDefs := map[string]v1beta1.TraitDefinition{ + "rollout": v1beta1.TraitDefinition{ + Spec: v1beta1.TraitDefinitionSpec{ + ManageWorkload: true, + }, + }, + "normal": v1beta1.TraitDefinition{ + Spec: v1beta1.TraitDefinitionSpec{}, + }, + } + appRev := v1beta1.ApplicationRevision{ + Spec: v1beta1.ApplicationRevisionSpec{ + TraitDefinitions: traitDefs, + }, + } + rolloutTrait := &unstructured.Unstructured{} + rolloutTrait.SetLabels(map[string]string{oam.TraitTypeLabel: "rollout"}) + + normalTrait := &unstructured.Unstructured{} + normalTrait.SetLabels(map[string]string{oam.TraitTypeLabel: "normal"}) + + workload := unstructured.Unstructured{} + workload.SetLabels(map[string]string{ + oam.WorkloadTypeLabel: "webservice", + }) + + comps := []*velatypes.ComponentManifest{ + { + Traits: []*unstructured.Unstructured{ + rolloutTrait, + normalTrait, + }, + StandardWorkload: &workload, + }, + } + + HandleCheckManageWorkloadTrait(appRev, comps) + Expect(len(rolloutTrait.GetLabels())).Should(BeEquivalentTo(2)) + Expect(rolloutTrait.GetLabels()[oam.LabelManageWorkloadTrait]).Should(BeEquivalentTo("true")) + Expect(len(normalTrait.GetLabels())).Should(BeEquivalentTo(1)) + Expect(normalTrait.GetLabels()[oam.LabelManageWorkloadTrait]).Should(BeEquivalentTo("")) + }) +}) diff --git a/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope.go b/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope.go index cf2088cb4..33b1f747e 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope.go +++ b/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope.go @@ -29,6 +29,7 @@ import ( "github.com/pkg/errors" apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -370,7 +371,7 @@ func getAppConfigNameFromLabel(o metav1.Object) string { func getVersioningPeerWorkloadRefs(ctx context.Context, c client.Reader, wlRef core.ObjectReference, ns string) ([]core.ObjectReference, error) { o := &unstructured.Unstructured{} o.SetGroupVersionKind(wlRef.GroupVersionKind()) - if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: wlRef.Name}, o); err != nil { + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: wlRef.Name}, o); err != nil && !apierrors.IsNotFound(err) { return nil, err } diff --git a/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope_controller.go b/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope_controller.go index 605056024..0dd7217fb 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope_controller.go +++ b/pkg/controller/core.oam.dev/v1alpha2/core/scopes/healthscope/healthscope_controller.go @@ -18,6 +18,7 @@ package healthscope import ( "context" + "encoding/json" "sort" "strings" "sync" @@ -615,12 +616,34 @@ func (r *Reconciler) createWorkloadRefs(ctx context.Context, appRef v1alpha2.App }, o); err != nil { continue } - if labels := o.GetLabels(); labels != nil && labels[oam.WorkloadTypeLabel] != "" { - wlRefs = append(wlRefs, WorkloadReference{ - ObjectReference: rs.ObjectReference, - clusterName: rs.Cluster, - envName: decisionsMap[rs.Cluster], - }) + + if labels := o.GetLabels(); labels != nil { + if labels[oam.WorkloadTypeLabel] != "" { + wlRefs = append(wlRefs, WorkloadReference{ + ObjectReference: rs.ObjectReference, + clusterName: rs.Cluster, + envName: decisionsMap[rs.Cluster], + }) + } else if labels[oam.TraitTypeLabel] != "" && labels[oam.LabelManageWorkloadTrait] == "true" { + // this means this trait is a manage-Workload trait, get workload GVK and name for trait's annotation + objectRef := corev1.ObjectReference{} + err := json.Unmarshal([]byte(o.GetAnnotations()[oam.AnnotationWorkloadGVK]), &objectRef) + if err != nil { + // don't break whole check process due to this error + continue + } + if o.GetAnnotations() != nil && len(o.GetAnnotations()[oam.AnnotationWorkloadName]) != 0 { + objectRef.Name = o.GetAnnotations()[oam.AnnotationWorkloadName] + } else { + // use component name as default + objectRef.Name = labels[oam.LabelAppComponent] + } + wlRefs = append(wlRefs, WorkloadReference{ + ObjectReference: objectRef, + clusterName: rs.Cluster, + envName: decisionsMap[rs.Cluster], + }) + } } } } diff --git a/pkg/controller/standard.oam.dev/v1alpha1/rollout/rollout_controller.go b/pkg/controller/standard.oam.dev/v1alpha1/rollout/rollout_controller.go index b3c230315..2eae7a01d 100644 --- a/pkg/controller/standard.oam.dev/v1alpha1/rollout/rollout_controller.go +++ b/pkg/controller/standard.oam.dev/v1alpha1/rollout/rollout_controller.go @@ -18,6 +18,7 @@ package rollout import ( "context" + "encoding/json" "github.com/pkg/errors" @@ -34,6 +35,8 @@ import ( common2 "github.com/oam-dev/kubevela/pkg/controller/common" rolloutplan "github.com/oam-dev/kubevela/pkg/controller/common/rollout" oamctrl "github.com/oam-dev/kubevela/pkg/controller/core.oam.dev" + + "github.com/oam-dev/kubevela/pkg/oam" oamutil "github.com/oam-dev/kubevela/pkg/oam/util" "github.com/oam-dev/kubevela/pkg/utils/apply" ) @@ -110,6 +113,20 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, err } + if rollout.Status.RollingState == v1alpha1.LocatingTargetAppState { + if rollout.GetAnnotations() == nil || rollout.GetAnnotations()[oam.AnnotationWorkloadName] != h.targetWorkload.GetName() { + gvk := map[string]string{"apiVersion": h.targetWorkload.GetAPIVersion(), "kind": h.targetWorkload.GetKind()} + gvkValue, _ := json.Marshal(gvk) + rollout.SetAnnotations(oamutil.MergeMapOverrideWithDst(rollout.GetAnnotations(), + map[string]string{oam.AnnotationWorkloadName: h.targetWorkload.GetName(), oam.AnnotationWorkloadGVK: string(gvkValue)})) + klog.InfoS("rollout controller set targetWorkload ", h.targetWorkload.GetName(), + "in annotation in rollout namespace: ", rollout.Namespace, " name", rollout.Name, "gvk", gvkValue) + // exit current reconcile before create target workload, this reconcile don't update status just modify annotation + // next round reconcile will create workload and pass `LocatingTargetAppState` phase + return ctrl.Result{}, h.Update(ctx, rollout) + } + } + switch rollout.Status.RollingState { case v1alpha1.RolloutDeletingState: removed, err := h.checkWorkloadNotExist(ctx) diff --git a/pkg/oam/labels.go b/pkg/oam/labels.go index f879b5e2b..a1bed0740 100644 --- a/pkg/oam/labels.go +++ b/pkg/oam/labels.go @@ -125,4 +125,10 @@ const ( // AnnotationLastAppliedConfiguration is kubectl annotations for 3-way merge AnnotationLastAppliedConfiguration = "kubectl.kubernetes.io/last-applied-configuration" + + // AnnotationWorkloadGVK indicates the managed workload's GVK by trait + AnnotationWorkloadGVK = "trait.oam.dev/workload-gvk" + + // AnnotationWorkloadName indicates the managed workload's name by trait + AnnotationWorkloadName = "trait.oam.dev/workload-name" ) diff --git a/runtime/rollout/cmd/main.go b/runtime/rollout/cmd/main.go index b24e20d59..fb04eda86 100644 --- a/runtime/rollout/cmd/main.go +++ b/runtime/rollout/cmd/main.go @@ -66,7 +66,7 @@ func main() { "Determines the namespace in which the leader election configmap will be created.") flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - flag.StringVar(&healthAddr, "health-addr", ":9440", "The address the health endpoint binds to.") + flag.StringVar(&healthAddr, "health-addr", ":19440", "The address the health endpoint binds to.") flag.Parse() // setup logging diff --git a/runtime/rollout/e2e/Dockerfile.e2e b/runtime/rollout/e2e/Dockerfile.e2e new file mode 100644 index 000000000..82c13c5b8 --- /dev/null +++ b/runtime/rollout/e2e/Dockerfile.e2e @@ -0,0 +1,44 @@ +# Build the manager binary +FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.16-alpine as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY ./tmp/go.mod go.mod +COPY ./tmp/go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY ./tmp/main.go main.go +COPY ./tmp/apis apis/ +COPY ./tmp/pkg pkg/ +COPY ./tmp/version version/ + +# Build +ARG TARGETARCH +ARG VERSION +ARG GITVERSION +RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \ + go build -a -ldflags "-s -w -X github.com/oam-dev/kubevela/version.VelaVersion=${VERSION:-undefined} -X github.com/oam-dev/kubevela/version.GitRevision=${GITVERSION:-undefined}" \ + -o manager-${TARGETARCH} main.go + +# Use alpine as base image due to the discussion in issue #1448 +# You can replace distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +# Overwrite `BASE_IMAGE` by passing `--build-arg=BASE_IMAGE=gcr.io/distroless/static:nonroot` +ARG BASE_IMAGE +FROM ${BASE_IMAGE:-alpine:latest} +# This is required by daemon connnecting with cri +RUN apk add --no-cache ca-certificates bash + +WORKDIR / + +ARG TARGETARCH +COPY --from=builder /workspace/manager-${TARGETARCH} /usr/local/bin/manager + +COPY ./tmp/entrypoint.sh /usr/local/bin/ + +ENTRYPOINT ["entrypoint.sh"] + +CMD ["manager"] diff --git a/test/e2e-multicluster-test/multicluster_rollout_test.go b/test/e2e-multicluster-test/multicluster_rollout_test.go new file mode 100644 index 000000000..3a0c4cb16 --- /dev/null +++ b/test/e2e-multicluster-test/multicluster_rollout_test.go @@ -0,0 +1,213 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_multicluster_test + +import ( + "context" + "fmt" + "io/ioutil" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/apis/standard.oam.dev/v1alpha1" + "github.com/oam-dev/kubevela/pkg/oam/util" + + "sigs.k8s.io/yaml" +) + +var _ = Describe("Test MultiClustet Rollout", func() { + Context("Test Runtime Cluster Rollout", func() { + var namespace string + var hubCtx context.Context + var workerCtx context.Context + var rollout v1alpha1.Rollout + var componentName string + var targetDeploy appsv1.Deployment + var sourceDeploy appsv1.Deployment + + BeforeEach(func() { + hubCtx, workerCtx, namespace = initializeContextAndNamespace() + componentName = "hello-world-server" + }) + + AfterEach(func() { + cleanUpNamespace(hubCtx, workerCtx, namespace) + ns := v1.Namespace{} + Eventually(func() error { return k8sClient.Get(hubCtx, types.NamespacedName{Name: namespace}, &ns) }, 300*time.Second, 300*time.Millisecond).Should(util.NotFoundMatcher{}) + }) + + verifySucceed := func(componentRevision string) { + By("check rollout status have succeed") + Eventually(func() error { + rolloutKey := types.NamespacedName{Namespace: namespace, Name: componentName} + if err := k8sClient.Get(workerCtx, rolloutKey, &rollout); err != nil { + return err + } + if rollout.Spec.TargetRevisionName != componentRevision { + return fmt.Errorf("rollout have not point to right targetRevision") + } + if rollout.Status.RollingState != v1alpha1.RolloutSucceedState { + return fmt.Errorf("error rollout status state %s", rollout.Status.RollingState) + } + compRevName := rollout.Spec.TargetRevisionName + deployKey := types.NamespacedName{Namespace: namespace, Name: compRevName} + if err := k8sClient.Get(workerCtx, deployKey, &targetDeploy); err != nil { + return err + } + if *targetDeploy.Spec.Replicas != *rollout.Spec.RolloutPlan.TargetSize { + return fmt.Errorf("targetDeploy replicas missMatch %d, %d", targetDeploy.Spec.Replicas, rollout.Spec.RolloutPlan.TargetSize) + } + if targetDeploy.Status.UpdatedReplicas != *targetDeploy.Spec.Replicas { + return fmt.Errorf("update not finish") + } + if len(targetDeploy.OwnerReferences) != 1 { + return fmt.Errorf("workload ownerReference missMatch") + } + // guarantee rollout's owners and workload's owners are same + if targetDeploy.OwnerReferences[0].Kind != rollout.OwnerReferences[0].Kind || + targetDeploy.OwnerReferences[0].Name != rollout.OwnerReferences[0].Name { + return fmt.Errorf("workload ownerReference missMatch") + } + if rollout.Status.LastSourceRevision == "" { + return nil + } + deployKey = types.NamespacedName{Namespace: namespace, Name: rollout.Status.LastSourceRevision} + if err := k8sClient.Get(workerCtx, deployKey, &sourceDeploy); err == nil || !apierrors.IsNotFound(err) { + return fmt.Errorf("source deploy still exist") + } + return nil + }, time.Second*360, 300*time.Millisecond).Should(BeNil()) + } + + It("Test Rollout whole feature in runtime cluster ", func() { + app := &v1beta1.Application{} + appYaml, err := ioutil.ReadFile("./testdata/app/app-rollout-envbinding.yaml") + Expect(err).Should(Succeed()) + Expect(yaml.Unmarshal([]byte(appYaml), app)).Should(Succeed()) + app.SetNamespace(namespace) + err = k8sClient.Create(hubCtx, app) + Expect(err).Should(Succeed()) + verifySucceed(componentName + "-v1") + + By("update application to v2") + checkApp := &v1beta1.Application{} + Eventually(func() error { + if err := k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp); err != nil { + return err + } + checkApp.Spec.Components[0].Properties.Raw = []byte(`{"image": "stefanprodan/podinfo:5.0.2"}`) + if err := k8sClient.Update(hubCtx, checkApp); err != nil { + return err + } + return nil + }, 500*time.Millisecond, 30*time.Second).Should(BeNil()) + verifySucceed(componentName + "-v2") + + By("revert to v1, should guarantee compRev v1 still exist") + appYaml, err = ioutil.ReadFile("./testdata/app/revert-app-envbinding.yaml") + Expect(err).Should(Succeed()) + + Expect(k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp)).Should(BeNil()) + revertApp := &v1beta1.Application{} + Expect(yaml.Unmarshal([]byte(appYaml), revertApp)).Should(Succeed()) + revertApp.SetNamespace(namespace) + revertApp.SetResourceVersion(checkApp.ResourceVersion) + + Eventually(func() error { + if err := k8sClient.Update(hubCtx, revertApp); err != nil { + return err + } + return nil + }, 500*time.Millisecond, 30*time.Second).Should(BeNil()) + verifySucceed(componentName + "-v1") + }) + + It("Test Rollout with health check policy, guarantee health scope controller work ", func() { + app := &v1beta1.Application{} + appYaml, err := ioutil.ReadFile("./testdata/app/multi-cluster-health-policy.yaml") + Expect(err).Should(Succeed()) + Expect(yaml.Unmarshal([]byte(appYaml), app)).Should(Succeed()) + app.SetNamespace(namespace) + err = k8sClient.Create(hubCtx, app) + Expect(err).Should(Succeed()) + verifySucceed(componentName + "-v1") + Eventually(func() error { + checkApp := v1beta1.Application{} + if err := k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: app.Name}, &checkApp); err != nil { + return err + } + if len(checkApp.Status.Services) == 0 { + return fmt.Errorf("app status service haven't write back") + } + compStatus := checkApp.Status.Services[0] + if compStatus.Env != "staging" { + return fmt.Errorf("comp status env miss-match") + } + if !compStatus.Healthy { + return fmt.Errorf("comp status not healthy") + } + if !strings.Contains(compStatus.Message, "Ready:2/2") { + return fmt.Errorf("comp status workload check don't work") + } + return nil + }, 300*time.Millisecond, 30*time.Second).Should(BeNil()) + By("update application to v2") + checkApp := &v1beta1.Application{} + Eventually(func() error { + if err := k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: app.Name}, checkApp); err != nil { + return err + } + checkApp.Spec.Components[0].Properties.Raw = []byte(`{"image": "stefanprodan/podinfo:5.0.2"}`) + if err := k8sClient.Update(hubCtx, checkApp); err != nil { + return err + } + return nil + }, 500*time.Millisecond, 30*time.Second).Should(BeNil()) + verifySucceed(componentName + "-v2") + Eventually(func() error { + // Note: KubeVela will only check the workload of the target revision + checkApp := v1beta1.Application{} + if err := k8sClient.Get(hubCtx, types.NamespacedName{Namespace: namespace, Name: app.Name}, &checkApp); err != nil { + return err + } + if len(checkApp.Status.Services) == 0 { + return fmt.Errorf("app status service haven't write back") + } + compStatus := checkApp.Status.Services[0] + if compStatus.Env != "staging" { + return fmt.Errorf("comp status env miss-match") + } + if !compStatus.Healthy { + return fmt.Errorf("comp status not healthy") + } + if !strings.Contains(compStatus.Message, "Ready:2/2") { + return fmt.Errorf("comp status workload check don't work") + } + return nil + }, 300*time.Millisecond, 30*time.Second).Should(BeNil()) + }) + }) +}) diff --git a/test/e2e-multicluster-test/testdata/app/app-rollout-envbinding.yaml b/test/e2e-multicluster-test/testdata/app/app-rollout-envbinding.yaml new file mode 100644 index 000000000..4cb5a6765 --- /dev/null +++ b/test/e2e-multicluster-test/testdata/app/app-rollout-envbinding.yaml @@ -0,0 +1,37 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: example-app + namespace: default +spec: + components: + - name: hello-world-server + type: webservice + properties: + image: stefanprodan/podinfo:4.0.3 + traits: + - type: rollout + properties: + targetSize: 2 + rolloutBatches: + - replicas: 1 + - replicas: 1 + + policies: + - name: example-multi-env-policy + type: env-binding + properties: + envs: + - name: staging + placement: # selecting the cluster to deploy to + clusterSelector: + name: cluster-worker + + workflow: + steps: + # deploy to staging env + - name: deploy-staging + type: deploy2env + properties: + policy: example-multi-env-policy + env: staging diff --git a/test/e2e-multicluster-test/testdata/app/multi-cluster-health-policy.yaml b/test/e2e-multicluster-test/testdata/app/multi-cluster-health-policy.yaml new file mode 100644 index 000000000..70ff93162 --- /dev/null +++ b/test/e2e-multicluster-test/testdata/app/multi-cluster-health-policy.yaml @@ -0,0 +1,48 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: example-app-rollout + namespace: default +spec: + components: + - name: hello-world-server + type: webservice + properties: + image: crccheck/hello-world + port: 8000 + type: webservice + traits: + - type: rollout + properties: + targetSize: 2 + rolloutBatches: + - replicas: 1 + - replicas: 1 + + policies: + - name: example-multi-env-policy + type: env-binding + properties: + envs: + - name: staging + placement: # 选择要部署的集群，并执行默认的发布策略 + clusterSelector: + name: cluster-worker + + + - name: health-policy-demo + type: health + properties: + probeInterval: 5 + probeTimeout: 10 + + + + workflow: + steps: + # 部署到预发环境中 + - name: deploy-staging + type: deploy2env + properties: + policy: example-multi-env-policy + env: staging diff --git a/test/e2e-multicluster-test/testdata/app/revert-app-envbinding.yaml b/test/e2e-multicluster-test/testdata/app/revert-app-envbinding.yaml new file mode 100644 index 000000000..d64b0a128 --- /dev/null +++ b/test/e2e-multicluster-test/testdata/app/revert-app-envbinding.yaml @@ -0,0 +1,38 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: example-app + namespace: default +spec: + components: + - name: hello-world-server + type: webservice + properties: + image: stefanprodan/podinfo:5.0.2 + traits: + - type: rollout + properties: + targetRevision: hello-world-server-v1 + targetSize: 2 + rolloutBatches: + - replicas: 1 + - replicas: 1 + + policies: + - name: example-multi-env-policy + type: env-binding + properties: + envs: + - name: staging + placement: # selecting the cluster to deploy to + clusterSelector: + name: cluster-worker + + workflow: + steps: + # deploy to staging env + - name: deploy-staging + type: deploy2env + properties: + policy: example-multi-env-policy + env: staging diff --git a/test/e2e-test/rollout_trait_test.go b/test/e2e-test/rollout_trait_test.go index 26906b009..a3ee7df74 100644 --- a/test/e2e-test/rollout_trait_test.go +++ b/test/e2e-test/rollout_trait_test.go @@ -18,9 +18,12 @@ package controllers_test import ( "context" + "encoding/json" "fmt" "time" + "github.com/oam-dev/kubevela/pkg/oam" + "sigs.k8s.io/yaml" v1 "k8s.io/api/apps/v1" @@ -113,10 +116,22 @@ var _ = Describe("rollout related e2e-test,rollout trait test", func() { return fmt.Errorf("error rollout status state %s", rollout.Status.RollingState) } compRevName = rollout.Spec.TargetRevisionName + if rollout.GetAnnotations() == nil || rollout.GetAnnotations()[oam.AnnotationWorkloadName] != componentRevision { + return fmt.Errorf("target workload name annotation missmatch want %s acctually %s", + rollout.GetAnnotations()[oam.AnnotationWorkloadName], componentRevision) + } deployKey := types.NamespacedName{Namespace: namespaceName, Name: compRevName} if err := k8sClient.Get(ctx, deployKey, &targerDeploy); err != nil { return err } + gvkStr := rollout.GetAnnotations()[oam.AnnotationWorkloadGVK] + gvk := map[string]string{} + if err := json.Unmarshal([]byte(gvkStr), &gvk); err != nil { + return err + } + if gvk["apiVersion"] != "apps/v1" || gvk["kind"] != "Deployment" { + return fmt.Errorf("error targetWorkload gvk") + } if *targerDeploy.Spec.Replicas != *rollout.Spec.RolloutPlan.TargetSize { return fmt.Errorf("targetDeploy replicas missMatch %d, %d", targerDeploy.Spec.Replicas, rollout.Spec.RolloutPlan.TargetSize) }