Feat: add support for json-patch and json-merge-patch (#3406)

* Feat: add support for json-patch and json-merge-patch

Signed-off-by: Somefive <yd219913@alibaba-inc.com>

* Fix: add e2e test

Signed-off-by: Somefive <yd219913@alibaba-inc.com>

* Fix: refactor json-patch field

Signed-off-by: Somefive <yd219913@alibaba-inc.com>
This commit is contained in:
Somefive
2022-03-11 14:57:50 +08:00
committed by GitHub
parent b13132ceba
commit 598de21f67
13 changed files with 444 additions and 52 deletions
@@ -1,11 +1,11 @@
# Code generated by KubeVela templates. DO NOT EDIT. Please edit the original cue file.
# Definition source cue file: vela-templates/definitions/internal/patch.cue
# Definition source cue file: vela-templates/definitions/internal/json-merge-patch.cue
apiVersion: core.oam.dev/v1beta1
kind: TraitDefinition
metadata:
annotations:
definition.oam.dev/description: Patch the output directly.
name: patch
definition.oam.dev/description: Patch the output following Json Merge Patch strategy, following RFC 7396.
name: json-merge-patch
namespace: {{ include "systemDefinitionNamespace" . }}
spec:
appliesToWorkloads:
@@ -15,6 +15,6 @@ spec:
cue:
template: |
parameter: {...}
// +patchStrategy=open
// +patchStrategy=jsonMergePatch
patch: parameter
@@ -0,0 +1,20 @@
# Code generated by KubeVela templates. DO NOT EDIT. Please edit the original cue file.
# Definition source cue file: vela-templates/definitions/internal/json-patch.cue
apiVersion: core.oam.dev/v1beta1
kind: TraitDefinition
metadata:
annotations:
definition.oam.dev/description: Patch the output following Json Patch strategy, following RFC 6902.
name: json-patch
namespace: {{ include "systemDefinitionNamespace" . }}
spec:
appliesToWorkloads:
- '*'
podDisruptive: true
schematic:
cue:
template: |
parameter: operations: [...{...}]
// +patchStrategy=jsonPatch
patch: parameter
@@ -0,0 +1,20 @@
# Code generated by KubeVela templates. DO NOT EDIT. Please edit the original cue file.
# Definition source cue file: vela-templates/definitions/internal/json-merge-patch.cue
apiVersion: core.oam.dev/v1beta1
kind: TraitDefinition
metadata:
annotations:
definition.oam.dev/description: Patch the output following Json Merge Patch strategy, following RFC 7396.
name: json-merge-patch
namespace: {{ include "systemDefinitionNamespace" . }}
spec:
appliesToWorkloads:
- '*'
podDisruptive: true
schematic:
cue:
template: |
parameter: {...}
// +patchStrategy=jsonMergePatch
patch: parameter
@@ -0,0 +1,20 @@
# Code generated by KubeVela templates. DO NOT EDIT. Please edit the original cue file.
# Definition source cue file: vela-templates/definitions/internal/json-patch.cue
apiVersion: core.oam.dev/v1beta1
kind: TraitDefinition
metadata:
annotations:
definition.oam.dev/description: Patch the output following Json Patch strategy, following RFC 6902.
name: json-patch
namespace: {{ include "systemDefinitionNamespace" . }}
spec:
appliesToWorkloads:
- '*'
podDisruptive: true
schematic:
cue:
template: |
parameter: operations: [...{...}]
// +patchStrategy=jsonPatch
patch: parameter
@@ -0,0 +1,39 @@
apiVersion: core.oam.dev/v1beta1
kind: Application
metadata:
name: busybox
spec:
components:
- name: busybox
type: webservice
properties:
image: busybox
cmd: ["sleep", "86400"]
labels:
pod-label-key: pod-label-value
to-delete-label-key: to-delete-label-value
traits:
# the json merge patch can be used to add, replace and delete fields
# the following part will
# 1. add `deploy-label-key` to deployment labels
# 2. set deployment replicas to 3
# 3. set `pod-label-key` to `pod-label-modified-value` in pod labels
# 4. delete `to-delete-label-key` in pod labels
# 5. reset `containers` for pod
- type: json-merge-patch
properties:
metadata:
labels:
deploy-label-key: deploy-label-added-value
spec:
replicas: 3
template:
metadata:
labels:
pod-label-key: pod-label-modified-value
to-delete-label-key: null
spec:
containers:
- name: busybox-new
image: busybox:1.34
command: ["sleep", "864000"]
@@ -0,0 +1,44 @@
apiVersion: core.oam.dev/v1beta1
kind: Application
metadata:
name: busybox
spec:
components:
- name: busybox
type: webservice
properties:
image: busybox
cmd: ["sleep", "86400"]
labels:
pod-label-key: pod-label-value
to-delete-label-key: to-delete-label-value
traits:
# the json patch can be used to add, replace and delete fields
# the following part will
# 1. add `deploy-label-key` to deployment labels
# 2. set deployment replicas to 3
# 3. set `pod-label-key` to `pod-label-modified-value` in pod labels
# 4. delete `to-delete-label-key` in pod labels
# 5. add sidecar container for pod
- type: json-patch
properties:
operations:
- op: add
path: "/metadata"
value:
labels:
deploy-label-key: deploy-label-added-value
- op: add
path: "/spec/replicas"
value: 3
- op: replace
path: "/spec/template/metadata/labels/pod-label-key"
value: pod-label-modified-value
- op: remove
path: "/spec/template/metadata/labels/to-delete-label-key"
- op: add
path: "/spec/template/spec/containers/1"
value:
name: busybox-sidecar
image: busybox:1.34
command: ["sleep", "864000"]
+1 -6
View File
@@ -350,12 +350,7 @@ func (td *traitDef) Complete(ctx process.Context, abstractTemplate string, param
if err != nil {
return errors.WithMessagef(err, "invalid patch of trait %s", td.name)
}
if sets.IsOpenPatch(patcher) {
if err := base.Open(); err != nil {
return errors.WithMessagef(err, "cannot convert base to open struct")
}
}
if err := base.Unify(p); err != nil {
if err := base.Unify(p, sets.CreateUnifyOptionsForPatcher(patcher)...); err != nil {
return errors.WithMessagef(err, "invalid patch trait %s into workload", td.name)
}
+54 -12
View File
@@ -369,15 +369,18 @@ parameter: {
},
},
},
"patch trait with open merge": {
"patch trait with json merge patch": {
traitTemplate: `
parameter: {...}
// +patchStrategy=open
// +patchStrategy=jsonMergePatch
patch: parameter
`,
params: map[string]interface{}{
"spec": map[string]interface{}{
"replicas": 5,
"template": map[string]interface{}{
"spec": nil,
},
},
},
expWorkload: &unstructured.Unstructured{
@@ -392,16 +395,7 @@ patch: parameter
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{"app.oam.dev/component": "test"},
},
"spec": map[string]interface{}{
"containers": []interface{}{map[string]interface{}{
"envFrom": []interface{}{map[string]interface{}{
"configMapRef": map[string]interface{}{"name": "testgame-config"},
}},
"image": "website:0.1",
"name": "main",
"ports": []interface{}{map[string]interface{}{"containerPort": int64(443)}}},
}}}}},
}}}},
},
expAssObjs: map[string]runtime.Object{
"AuxiliaryWorkloadgameconfig": &unstructured.Unstructured{
@@ -412,6 +406,54 @@ patch: parameter
},
},
},
"patch trait with json patch": {
traitTemplate: `
parameter: {operations: [...{...}]}
// +patchStrategy=jsonPatch
patch: parameter
`,
params: map[string]interface{}{
"operations": []map[string]interface{}{
{"op": "replace", "path": "/spec/replicas", "value": 5},
{"op": "remove", "path": "/spec/template/spec"},
},
},
expWorkload: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"spec": map[string]interface{}{
"replicas": int64(5),
"selector": map[string]interface{}{
"matchLabels": map[string]interface{}{
"app.oam.dev/component": "test"}},
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{"app.oam.dev/component": "test"},
}}}},
},
expAssObjs: map[string]runtime.Object{
"AuxiliaryWorkloadgameconfig": &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": map[string]interface{}{"name": "testgame-config"}, "data": map[string]interface{}{"enemies": "enemies-data", "lives": "lives-data"}},
},
},
},
"patch trait with invalid json patch": {
traitTemplate: `
parameter: {patch: [...{...}]}
// +patchStrategy=jsonPatch
patch: parameter
`,
params: map[string]interface{}{
"patch": []map[string]interface{}{
{"op": "what", "path": "/spec/replicas", "value": 5},
},
},
hasCompileErr: true,
},
"patch trait with replace": {
traitTemplate: `
parameter: {
+3 -14
View File
@@ -35,9 +35,8 @@ type Instance interface {
String() string
Unstructured() (*unstructured.Unstructured, error)
IsBase() bool
Unify(other Instance) error
Unify(other Instance, options ...sets.UnifyOption) error
Compile() ([]byte, error)
Open() error
}
type instance struct {
@@ -89,8 +88,8 @@ func (inst *instance) Unstructured() (*unstructured.Unstructured, error) {
}
// Unify implement unity operations between instances
func (inst *instance) Unify(other Instance) error {
pv, err := sets.StrategyUnify(inst.v, other.String())
func (inst *instance) Unify(other Instance, options ...sets.UnifyOption) error {
pv, err := sets.StrategyUnify(inst.v, other.String(), options...)
if err != nil {
return err
}
@@ -98,16 +97,6 @@ func (inst *instance) Unify(other Instance) error {
return nil
}
// Open convert instance value to open
func (inst *instance) Open() error {
open, err := sets.OpenBaiscLit(inst.v)
if err != nil {
return err
}
inst.v = open
return nil
}
// NewBase create a base instance
func NewBase(v cue.Value) (Instance, error) {
vs, err := openPrint(v)
+114 -13
View File
@@ -22,6 +22,7 @@ import (
"cuelang.org/go/cue"
"cuelang.org/go/cue/ast"
"cuelang.org/go/cue/parser"
jsonpatch "github.com/evanphx/json-patch"
"github.com/pkg/errors"
)
@@ -33,16 +34,62 @@ const (
// StrategyRetainKeys notes on the strategic merge patch using the retainKeys strategy
StrategyRetainKeys = "retainKeys"
// StrategyOpen notes on the strategic merge patch will allow any merge
StrategyOpen = "open"
// StrategyReplace notes on the strategic merge patch will allow replacing list
StrategyReplace = "replace"
// StrategyJSONPatch notes on the strategic merge patch will follow the RFC 6902 to run JsonPatch
StrategyJSONPatch = "jsonPatch"
// StrategyJSONMergePatch notes on the strategic merge patch will follow the RFC 7396 to run JsonMergePatch
StrategyJSONMergePatch = "jsonMergePatch"
)
var (
notFoundErr = errors.Errorf("not found")
)
// UnifyParams params for unify
type UnifyParams struct {
PatchStrategy string
}
// UnifyOption defines the option for unify
type UnifyOption interface {
ApplyToOption(params *UnifyParams)
}
// UnifyByJSONPatch unify by json patch following RFC 6902
type UnifyByJSONPatch struct{}
// ApplyToOption apply to option
func (op UnifyByJSONPatch) ApplyToOption(params *UnifyParams) {
params.PatchStrategy = StrategyJSONPatch
}
// UnifyByJSONMergePatch unify by json patch following RFC 7396
type UnifyByJSONMergePatch struct{}
// ApplyToOption apply to option
func (op UnifyByJSONMergePatch) ApplyToOption(params *UnifyParams) {
params.PatchStrategy = StrategyJSONMergePatch
}
func newUnifyParams(options ...UnifyOption) *UnifyParams {
params := &UnifyParams{}
for _, op := range options {
op.ApplyToOption(params)
}
return params
}
// CreateUnifyOptionsForPatcher create unify options for patcher
func CreateUnifyOptionsForPatcher(patcher cue.Value) (options []UnifyOption) {
if IsJSONPatch(patcher) {
options = append(options, UnifyByJSONPatch{})
} else if IsJSONMergePatch(patcher) {
options = append(options, UnifyByJSONMergePatch{})
}
return
}
type interceptor func(baseNode ast.Node, patchNode ast.Node) error
func listMergeProcess(field *ast.Field, key string, baseList, patchList *ast.ListLit) {
@@ -185,19 +232,30 @@ func isStrategyRetainKeys(node *ast.Field) bool {
return false
}
// IsOpenPatch check if patcher has open annotation
func IsOpenPatch(patcher cue.Value) bool {
// IsJSONMergePatch check if patcher is json merge patch
func IsJSONMergePatch(patcher cue.Value) bool {
tags := findCommentTag(patcher.Doc())
for tk, tv := range tags {
if tk == TagPatchStrategy && tv == StrategyOpen {
return true
}
}
return false
return tags[TagPatchStrategy] == StrategyJSONMergePatch
}
// IsJSONPatch check if patcher is json patch
func IsJSONPatch(patcher cue.Value) bool {
tags := findCommentTag(patcher.Doc())
return tags[TagPatchStrategy] == StrategyJSONPatch
}
// StrategyUnify unify the objects by the strategy
func StrategyUnify(base, patch string) (string, error) {
func StrategyUnify(base, patch string, options ...UnifyOption) (ret string, err error) {
params := newUnifyParams(options...)
var patchOpts []interceptor
if params.PatchStrategy == StrategyJSONMergePatch || params.PatchStrategy == StrategyJSONPatch {
base, err = OpenBaiscLit(base)
if err != nil {
return base, err
}
} else {
patchOpts = []interceptor{strategyPatchHandle()}
}
baseFile, err := parser.ParseFile("-", base, parser.ParseComments)
if err != nil {
return "", errors.WithMessage(err, "invalid base cue file")
@@ -207,10 +265,10 @@ func StrategyUnify(base, patch string) (string, error) {
return "", errors.WithMessage(err, "invalid patch cue file")
}
return strategyUnify(baseFile, patchFile, strategyPatchHandle())
return strategyUnify(baseFile, patchFile, params, patchOpts...)
}
func strategyUnify(baseFile *ast.File, patchFile *ast.File, patchOpts ...interceptor) (string, error) {
func strategyUnify(baseFile *ast.File, patchFile *ast.File, params *UnifyParams, patchOpts ...interceptor) (string, error) {
for _, option := range patchOpts {
if err := option(baseFile, patchFile); err != nil {
return "", errors.WithMessage(err, "process patchOption")
@@ -228,6 +286,12 @@ func strategyUnify(baseFile *ast.File, patchFile *ast.File, patchOpts ...interce
return "", errors.WithMessage(err, "compile patch file")
}
if params.PatchStrategy == StrategyJSONMergePatch {
return jsonMergePatch(baseInst.Value(), patchInst.Value())
} else if params.PatchStrategy == StrategyJSONPatch {
return jsonPatch(baseInst.Value(), patchInst.Lookup("operations"))
}
ret := baseInst.Value().Unify(patchInst.Value())
rv, err := toString(ret)
@@ -272,3 +336,40 @@ func findCommentTag(commentGroup []*ast.CommentGroup) map[string]string {
}
return kval
}
func jsonMergePatch(base cue.Value, patch cue.Value) (string, error) {
baseJSON, err := base.MarshalJSON()
if err != nil {
return "", errors.Wrapf(err, "failed to marshal base value")
}
patchJSON, err := patch.MarshalJSON()
if err != nil {
return "", errors.Wrapf(err, "failed to marshal patch value")
}
merged, err := jsonpatch.MergePatch(baseJSON, patchJSON)
if err != nil {
return "", errors.Wrapf(err, "failed to merge base value and patch value by JsonMergePatch")
}
return string(merged), nil
}
func jsonPatch(base cue.Value, patch cue.Value) (string, error) {
baseJSON, err := base.MarshalJSON()
if err != nil {
return "", errors.Wrapf(err, "failed to marshal base value")
}
patchJSON, err := patch.MarshalJSON()
if err != nil {
return "", errors.Wrapf(err, "failed to marshal patch value")
}
decodedPatch, err := jsonpatch.DecodePatch(patchJSON)
if err != nil {
return "", errors.Wrapf(err, "failed to decode patch")
}
merged, err := decodedPatch.Apply(baseJSON)
if err != nil {
return "", errors.Wrapf(err, "failed to apply json patch")
}
return string(merged), nil
}
+107
View File
@@ -0,0 +1,107 @@
/*
Copyright 2021 The KubeVela Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers_test
import (
"context"
"io/ioutil"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/pointer"
"sigs.k8s.io/yaml"
"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
)
func readAppFromFile(filename string) (*v1beta1.Application, error) {
bs, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
app := &v1beta1.Application{}
if err = yaml.Unmarshal(bs, app); err != nil {
return nil, err
}
return app, nil
}
var _ = Describe("Trait tests", func() {
ctx := context.Background()
var namespace string
BeforeEach(func() {
namespace = randomNamespaceName("trait-test")
Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: v1.ObjectMeta{Name: namespace}})).Should(Succeed())
})
AfterEach(func() {
ns := &corev1.Namespace{}
Expect(k8sClient.Get(ctx, types.NamespacedName{Name: namespace}, ns)).Should(Succeed())
Expect(k8sClient.Delete(ctx, ns)).Should(Succeed())
})
Context("Test app with traits", func() {
It("Test json-patch trait", func() {
app, err := readAppFromFile("../../docs/examples/traits/json-patch/example.yaml")
Expect(err).Should(Succeed())
app.SetNamespace(namespace)
Expect(k8sClient.Create(ctx, app)).Should(Succeed())
Eventually(func(g Gomega) {
deploy := &appsv1.Deployment{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "busybox"}, deploy)).Should(Succeed())
g.Expect(deploy.Labels).ShouldNot(BeNil())
g.Expect(deploy.Labels["deploy-label-key"]).Should(Equal("deploy-label-added-value"))
g.Expect(deploy.Spec.Replicas).Should(Equal(pointer.Int32(3)))
g.Expect(deploy.Spec.Template.ObjectMeta.Labels).ShouldNot(BeNil())
g.Expect(deploy.Spec.Template.ObjectMeta.Labels["pod-label-key"]).Should(Equal("pod-label-modified-value"))
g.Expect(deploy.Spec.Template.ObjectMeta.Labels["to-delete-label-key"]).ShouldNot(Equal("to-delete-label-value"))
g.Expect(len(deploy.Spec.Template.Spec.Containers)).Should(Equal(2))
g.Expect(deploy.Spec.Template.Spec.Containers[1].Name).Should(Equal("busybox-sidecar"))
g.Expect(deploy.Spec.Template.Spec.Containers[1].Image).Should(Equal("busybox:1.34"))
g.Expect(deploy.Spec.Template.Spec.Containers[1].Command).Should(Equal([]string{"sleep", "864000"}))
}, 15*time.Second).Should(Succeed())
})
It("Test json-merge-patch trait", func() {
app, err := readAppFromFile("../../docs/examples/traits/json-merge-patch/example.yaml")
Expect(err).Should(Succeed())
app.SetNamespace(namespace)
Expect(k8sClient.Create(ctx, app)).Should(Succeed())
Eventually(func(g Gomega) {
deploy := &appsv1.Deployment{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: "busybox"}, deploy)).Should(Succeed())
g.Expect(deploy.Labels).ShouldNot(BeNil())
g.Expect(deploy.Labels["deploy-label-key"]).Should(Equal("deploy-label-added-value"))
g.Expect(deploy.Spec.Replicas).Should(Equal(pointer.Int32(3)))
g.Expect(deploy.Spec.Template.ObjectMeta.Labels).ShouldNot(BeNil())
g.Expect(deploy.Spec.Template.ObjectMeta.Labels["pod-label-key"]).Should(Equal("pod-label-modified-value"))
_, exists := deploy.Spec.Template.ObjectMeta.Labels["to-delete-label-key"]
g.Expect(exists).Should(BeFalse())
g.Expect(len(deploy.Spec.Template.Spec.Containers)).Should(Equal(1))
g.Expect(deploy.Spec.Template.Spec.Containers[0].Name).Should(Equal("busybox-new"))
g.Expect(deploy.Spec.Template.Spec.Containers[0].Image).Should(Equal("busybox:1.34"))
g.Expect(deploy.Spec.Template.Spec.Containers[0].Command).Should(Equal([]string{"sleep", "864000"}))
}, 15*time.Second).Should(Succeed())
})
})
})
@@ -1,8 +1,8 @@
patch: {
"json-merge-patch": {
type: "trait"
annotations: {}
labels: {}
description: "Patch the output directly."
description: "Patch the output following Json Merge Patch strategy, following RFC 7396."
attributes: {
podDisruptive: true
appliesToWorkloads: ["*"]
@@ -10,6 +10,6 @@ patch: {
}
template: {
parameter: {...}
// +patchStrategy=open
// +patchStrategy=jsonMergePatch
patch: parameter
}
@@ -0,0 +1,15 @@
"json-patch": {
type: "trait"
annotations: {}
labels: {}
description: "Patch the output following Json Patch strategy, following RFC 6902."
attributes: {
podDisruptive: true
appliesToWorkloads: ["*"]
}
}
template: {
parameter: operations: [...{...}]
// +patchStrategy=jsonPatch
patch: parameter
}