From 133a89be3fb6e989bf21d83fdd976d2c92c46a7a Mon Sep 17 00:00:00 2001 From: "Jian.Li" Date: Thu, 16 Dec 2021 17:53:15 +0800 Subject: [PATCH] Fix: deprecate CRD discovery for CUE import in Definition to prevent memory leak and OOM crash (#2925) * Pause test case Signed-off-by: Jian.Li * fix lint Signed-off-by: Jian.Li * diable discover the open api of the CRD Signed-off-by: Jian.Li * fix definition test cases Signed-off-by: Jian.Li --- .../application_controller_test.go | 2 +- .../componentdefinition_controller_test.go | 6 +-- .../traitdefinition_controller_test.go | 6 +-- pkg/controller/utils/utils.go | 6 +-- pkg/cue/package_suit_test.go | 6 +-- pkg/cue/packages/package.go | 12 +++--- .../appfile/dryrun/testdata/cd-myworker.yaml | 8 ++-- .../dryrun/testdata/diff-apprevision.yaml | 40 +++++++++---------- .../appfile/dryrun/testdata/td-myingress.yaml | 15 +++---- 9 files changed, 48 insertions(+), 53 deletions(-) diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go index 9e52b67ad..97f17abcf 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go @@ -1280,7 +1280,7 @@ var _ = Describe("Test Application Controller", func() { }, appRevision)).Should(BeNil()) }) - It("app-import-pkg will create workload by imported kube package", func() { + PIt("app-import-pkg will create workload by imported kube package", func() { ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/controller/core.oam.dev/v1alpha2/core/components/componentdefinition/componentdefinition_controller_test.go b/pkg/controller/core.oam.dev/v1alpha2/core/components/componentdefinition/componentdefinition_controller_test.go index 0ba6a07cc..255a69ef2 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/core/components/componentdefinition/componentdefinition_controller_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/core/components/componentdefinition/componentdefinition_controller_test.go @@ -706,11 +706,9 @@ spec: schematic: cue: template: | - import ( - ev1 "example.com/v1" - ) - output: ev1.#Foo output: { + kind: "Foo" + apiVersion: "example.com/v1" spec: key: parameter.key1 status: key: parameter.key2 } diff --git a/pkg/controller/core.oam.dev/v1alpha2/core/traits/traitdefinition/traitdefinition_controller_test.go b/pkg/controller/core.oam.dev/v1alpha2/core/traits/traitdefinition/traitdefinition_controller_test.go index e621e3621..930d56734 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/core/traits/traitdefinition/traitdefinition_controller_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/core/traits/traitdefinition/traitdefinition_controller_test.go @@ -268,11 +268,9 @@ spec: schematic: cue: template: | - import ( - ev1 "example.com/v1" - ) - output: ev1.#Foo output: { + kind: "Foo" + apiVersion: "example.com/v1" spec: key: parameter.key1 status: key: parameter.key2 } diff --git a/pkg/controller/utils/utils.go b/pkg/controller/utils/utils.go index 4cb886123..eeed321b3 100644 --- a/pkg/controller/utils/utils.go +++ b/pkg/controller/utils/utils.go @@ -318,9 +318,9 @@ func RefreshPackageDiscover(ctx context.Context, k8sClient client.Client, dm dis } // Test whether the refresh is successful - if exist := pd.Exist(targetGVK); !exist { - return fmt.Errorf("get CRD %s error", targetGVK.String()) - } + // if exist := pd.Exist(targetGVK); !exist { + // return fmt.Errorf("get CRD %s error", targetGVK.String()) + // } return nil } diff --git a/pkg/cue/package_suit_test.go b/pkg/cue/package_suit_test.go index 3c74ec3ee..8eacb2691 100644 --- a/pkg/cue/package_suit_test.go +++ b/pkg/cue/package_suit_test.go @@ -50,7 +50,7 @@ import ( ) var _ = Describe("Package discovery resources for definition from K8s APIServer", func() { - It("check that all built-in k8s resource are registered", func() { + PIt("check that all built-in k8s resource are registered", func() { var localSchemeBuilder = runtime.SchemeBuilder{ admissionregistrationv1.AddToScheme, appsv1.AddToScheme, @@ -95,7 +95,7 @@ var _ = Describe("Package discovery resources for definition from K8s APIServer" } }) - It("discovery built-in k8s resource with kube prefix", func() { + PIt("discovery built-in k8s resource with kube prefix", func() { By("test ingress in kube package") bi := build.NewContext().NewInstance("", nil) @@ -388,7 +388,7 @@ output: { }) - It("discovery built-in k8s resource with third-party path", func() { + PIt("discovery built-in k8s resource with third-party path", func() { By("test ingress in kube package") bi := build.NewContext().NewInstance("", nil) diff --git a/pkg/cue/packages/package.go b/pkg/cue/packages/package.go index bddc1dd04..22736a334 100644 --- a/pkg/cue/packages/package.go +++ b/pkg/cue/packages/package.go @@ -17,7 +17,6 @@ limitations under the License. package packages import ( - "context" "fmt" "path/filepath" "strings" @@ -131,11 +130,12 @@ func (pd *PackageDiscover) ListPackageKinds() map[string][]VersionKind { // RefreshKubePackagesFromCluster will use K8s client to load/refresh all K8s open API as a reference kube package using in template func (pd *PackageDiscover) RefreshKubePackagesFromCluster() error { - body, err := pd.client.Get().AbsPath("/openapi/v2").Do(context.Background()).Raw() - if err != nil { - return err - } - return pd.addKubeCUEPackagesFromCluster(string(body)) + return nil + // body, err := pd.client.Get().AbsPath("/openapi/v2").Do(context.Background()).Raw() + // if err != nil { + // return err + // } + // return pd.addKubeCUEPackagesFromCluster(string(body)) } // Exist checks if the GVK exists in the built-in packages diff --git a/references/appfile/dryrun/testdata/cd-myworker.yaml b/references/appfile/dryrun/testdata/cd-myworker.yaml index 1e374f7c0..57ce4ae47 100644 --- a/references/appfile/dryrun/testdata/cd-myworker.yaml +++ b/references/appfile/dryrun/testdata/cd-myworker.yaml @@ -10,10 +10,10 @@ spec: schematic: cue: template: | - import ( - apps "kube/apps/v1" - ) - output: apps.#Deployment + output: { + apiVersion: "apps/v1" + kind: "Deployment" + } output: { spec: { selector: matchLabels: { diff --git a/references/appfile/dryrun/testdata/diff-apprevision.yaml b/references/appfile/dryrun/testdata/diff-apprevision.yaml index dbeb317cd..f01d1c39e 100644 --- a/references/appfile/dryrun/testdata/diff-apprevision.yaml +++ b/references/appfile/dryrun/testdata/diff-apprevision.yaml @@ -117,15 +117,14 @@ spec: spec: schematic: cue: - template: "import (\n apps \"kube/apps/v1\"\n)\noutput: apps.#Deployment\noutput: - {\n\tspec: {\n\t\tselector: matchLabels: {\n\t\t\t\"app.oam.dev/component\": - context.name\n\t\t}\n\n\t\ttemplate: {\n\t\t\tmetadata: labels: {\n\t\t\t\t\"app.oam.dev/component\": - context.name\n\t\t\t}\n\n\t\t\tspec: {\n\t\t\t\tcontainers: [{\n\t\t\t\t\tname: - \ context.name\n\t\t\t\t\timage: parameter.image\n\n\t\t\t\t\tif parameter[\"cmd\"] - != _|_ {\n\t\t\t\t\t\tcommand: parameter.cmd\n\t\t\t\t\t}\n\t\t\t\t}]\n\t\t\t}\n\t\t}\n\t}\n}\n\nparameter: - {\n\t// +usage=Which image would you like to use for your service\n\t// - +short=i\n\timage: string\n\t// +usage=Commands to run in the container\n\tcmd?: - [...string]\n}\n" + template: "output: {\n apiVersion: \"apps/v1\"\n kind: \"Deployment\"\n}\noutput: + {\n\tspec: {\n\t\tselector: matchLabels: {\n\t\t\t\"app.oam.dev/component\": + context.name\n\t\t}\n\n\t\ttemplate: {\n\t\t\tmetadata: labels: {\n\t\t\t\t\"app.oam.dev/component\": + context.name\n\t\t\t}\n\n\t\t\tspec: {\n\t\t\t\tcontainers: [{\n\t\t\t\t\tname: + \ context.name\n\t\t\t\t\timage: parameter.image\n\n\t\t\t\t\tif parameter[\"cmd\"] + != _|_ {\n\t\t\t\t\t\tcommand: parameter.cmd\n\t\t\t\t\t}\n\t\t\t\t}]\n\t\t\t}\n\t\t}\n\t}\n}\n\nparameter: + {\n\t// +usage=Which image would you like to use for your service\n\t// +short=i\n\timage: + string\n\t// +usage=Commands to run in the container\n\tcmd?: [...string]\n}\n" workload: definition: apiVersion: apps/v1 @@ -219,18 +218,17 @@ spec: name: "" schematic: cue: - template: "import (\n\tkubev1 \"kube/v1\"\n\tnetwork \"kube/networking.k8s.io/v1beta1\"\n)\n\nparameter: - {\n\tdomain: string\n\thttp: [string]: int\n}\n\noutputs: {\nservice: - kubev1.#Service\ningress: network.#Ingress\n}\n\n// trait template can - have multiple outputs in one trait\noutputs: service: {\n\tmetadata:\n\t\tname: - context.name\n\tspec: {\n\t\tselector:\n\t\t\t\"app.oam.dev/component\": - context.name\n\t\tports: [\n\t\t\tfor k, v in parameter.http {\n\t\t\t\tport: - \ v\n\t\t\t\ttargetPort: v\n\t\t\t},\n\t\t]\n\t}\n}\n\noutputs: - ingress: {\n\tmetadata:\n\t\tname: context.name\n\tspec: {\n\t\trules: - [{\n\t\t\thost: parameter.domain\n\t\t\thttp: {\n\t\t\t\tpaths: [\n\t\t\t\t\tfor - k, v in parameter.http {\n\t\t\t\t\t\tpath: k\n\t\t\t\t\t\tbackend: - {\n\t\t\t\t\t\t\tserviceName: context.name\n\t\t\t\t\t\t\tservicePort: - v\n\t\t\t\t\t\t}\n\t\t\t\t\t},\n\t\t\t\t]\n\t\t\t}\n\t\t}]\n\t}\n}\n" + template: "parameter: {\n\tdomain: string\n\thttp: [string]: int\n}\n\noutputs: + {\n service: {\n apiVersion: \"v1\"\n kind: \"Service\"\n }\n ingress: + {\n apiVersion: \"networking.k8s.io/v1beta1\"\n kind: \"Ingress\"\n + \ }\n}\n\n// trait template can have multiple outputs in one trait\noutputs: + service: {\n\tmetadata:\n\t\tname: context.name\n\tspec: {\n\t\tselector:\n\t\t\t\"app.oam.dev/component\": + context.name\n\t\tports: [\n\t\t\tfor k, v in parameter.http {\n\t\t\t\tport: + \ v\n\t\t\t\ttargetPort: v\n\t\t\t},\n\t\t]\n\t}\n}\n\noutputs: ingress: + {\n\tmetadata:\n\t\tname: context.name\n\tspec: {\n\t\trules: [{\n\t\t\thost: + parameter.domain\n\t\t\thttp: {\n\t\t\t\tpaths: [\n\t\t\t\t\tfor k, v in parameter.http + {\n\t\t\t\t\t\tpath: k\n\t\t\t\t\t\tbackend: {\n\t\t\t\t\t\t\tserviceName: + context.name\n\t\t\t\t\t\t\tservicePort: v\n\t\t\t\t\t\t}\n\t\t\t\t\t},\n\t\t\t\t]\n\t\t\t}\n\t\t}]\n\t}\n}\n" status: {} myscaler: apiVersion: core.oam.dev/v1beta1 diff --git a/references/appfile/dryrun/testdata/td-myingress.yaml b/references/appfile/dryrun/testdata/td-myingress.yaml index 6503830a5..65e74b39f 100644 --- a/references/appfile/dryrun/testdata/td-myingress.yaml +++ b/references/appfile/dryrun/testdata/td-myingress.yaml @@ -9,19 +9,20 @@ spec: schematic: cue: template: | - import ( - kubev1 "kube/v1" - network "kube/networking.k8s.io/v1beta1" - ) - parameter: { domain: string http: [string]: int } outputs: { - service: kubev1.#Service - ingress: network.#Ingress + service: { + apiVersion: "v1" + kind: "Service" + } + ingress: { + apiVersion: "networking.k8s.io/v1beta1" + kind: "Ingress" + } } // trait template can have multiple outputs in one trait