From 3d2bd9a89826bfba9314f5f1721f82222697cda7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=A9=E5=85=83?= Date: Tue, 13 Oct 2020 19:34:33 +0800 Subject: [PATCH] add a design doc for route trait, along with fix and tests --- Makefile | 1 + README.md | 2 + api/types/types.go | 5 +- api/v1alpha1/route_types.go | 53 ++- api/v1alpha1/zz_generated.deepcopy.go | 88 ++-- ...ore.oam.dev_applicationconfigurations.yaml | 2 - .../crds/standard.oam.dev_routes.yaml | 204 ++++----- .../templates/definitions/manualscalars.yaml | 2 +- .../templates/definitions/metricstraits.yaml | 2 +- .../templates/definitions/routetrait.yaml | 2 +- config/rbac/role.yaml | 20 - design/route.md | 126 ++++++ .../samples/route-demo/definition.yaml | 53 +++ documentation/samples/route-demo/deploy.yaml | 60 +++ .../samples/route-demo/deployment-app.yaml | 58 +++ .../samples/route-demo/webservice-app.yaml | 51 +++ go.mod | 2 +- go.sum | 2 + pkg/application/app.go | 12 +- pkg/commands/ls.go | 3 +- pkg/controller/v1alpha1/routes/ingress/api.go | 26 ++ .../v1alpha1/routes/ingress/api_test.go | 16 + .../v1alpha1/routes/ingress/nginx_ingress.go | 124 ++++++ .../routes/ingress/nginx_ingress_test.go | 109 +++++ .../v1alpha1/routes/route_controller.go | 388 +++++++----------- .../v1alpha1/routes/route_controller_test.go | 351 ++++++++++++++++ pkg/controller/v1alpha1/routes/suite_test.go | 107 ++++- pkg/controller/v1alpha1/routes/util.go | 161 ++++++++ pkg/controller/v1alpha1/routes/util_test.go | 1 + pkg/oam/trait.go | 18 - 30 files changed, 1591 insertions(+), 458 deletions(-) create mode 100644 design/route.md create mode 100644 documentation/samples/route-demo/definition.yaml create mode 100644 documentation/samples/route-demo/deploy.yaml create mode 100644 documentation/samples/route-demo/deployment-app.yaml create mode 100644 documentation/samples/route-demo/webservice-app.yaml create mode 100644 pkg/controller/v1alpha1/routes/ingress/api.go create mode 100644 pkg/controller/v1alpha1/routes/ingress/api_test.go create mode 100644 pkg/controller/v1alpha1/routes/ingress/nginx_ingress.go create mode 100644 pkg/controller/v1alpha1/routes/ingress/nginx_ingress_test.go create mode 100644 pkg/controller/v1alpha1/routes/route_controller_test.go create mode 100644 pkg/controller/v1alpha1/routes/util.go create mode 100644 pkg/controller/v1alpha1/routes/util_test.go diff --git a/Makefile b/Makefile index 311580f81..c4c17e7fb 100644 --- a/Makefile +++ b/Makefile @@ -128,6 +128,7 @@ core-uninstall: manifests # Generate manifests e.g. CRD, RBAC etc. manifests: controller-gen $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=charts/vela-core/crds + rm charts/vela-core/crds/_.yaml # Generate code generate: controller-gen diff --git a/README.md b/README.md index ac10375ee..69af32f1b 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,8 @@ Let's take `route` as example. ### `route` +If you want to use `route`, please make sure you have [nginx-ingress controller[https://kubernetes.github.io/ingress-nginx/deploy/] in your cluster. + ```console $ vela route mycomp --app myapp Adding route for app mycomp diff --git a/api/types/types.go b/api/types/types.go index 62b7e2911..a02ecd5ea 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -15,10 +15,7 @@ const ( AnnKind = "definition.oam.dev/kind" AnnDescription = "definition.oam.dev/description" - // Indicate which workloadDefinition generate from - AnnWorkloadDef = "workload.oam.dev/name" - // Indicate which traitDefinition generate from - AnnTraitDef = "trait.oam.dev/name" + LabelPodSpecable = "workload.oam.dev/podspecable" ) const ( diff --git a/api/v1alpha1/route_types.go b/api/v1alpha1/route_types.go index 9cb210b7a..621c70f08 100644 --- a/api/v1alpha1/route_types.go +++ b/api/v1alpha1/route_types.go @@ -19,7 +19,6 @@ package v1alpha1 import ( runtimev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1" "github.com/crossplane/oam-kubernetes-runtime/pkg/oam" - "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -34,15 +33,24 @@ type RouteSpec struct { // Host is the host of the route Host string `json:"host"` - // Path is location Path, default for "/" - Path string `json:"path,omitempty"` - // TLS indicate route trait will create SSL secret using cert-manager with specified issuer // If this is nil, route trait will use a selfsigned issuer TLS *TLS `json:"tls,omitempty"` - // DefaultBackend uses serviceName - DefaultBackend *v1beta1.IngressBackend `json:"defaultBackend,omitempty"` + // Rules contain multiple rules of route + Rules []Rule `json:"rules,omitempty"` + + // Provider indicate which ingress controller implementation the route trait will use, by default it's nginx-ingress + Provider string `json:"provider,omitempty"` +} + +// Rule defines to route rule +type Rule struct { + // Name will become the suffix of underlying ingress created by this rule, if not, will use index as suffix. + Name string `json:"name,omitempty"` + + // Path is location Path, default for "/" + Path string `json:"path,omitempty"` // RewriteTarget will rewrite request from Path to RewriteTarget path. RewriteTarget string `json:"rewriteTarget,omitempty"` @@ -50,6 +58,9 @@ type RouteSpec struct { // CustomHeaders pass a custom list of headers to the backend service. CustomHeaders map[string]string `json:"customHeaders,omitempty"` + // DefaultBackend will become the ingress default backend if the backend is not available + DefaultBackend *runtimev1alpha1.TypedReference `json:"defaultBackend,omitempty"` + // Backend indicate how to connect backend service // If it's nil, will auto discovery Backend *Backend `json:"backend,omitempty"` @@ -58,7 +69,8 @@ type RouteSpec struct { type TLS struct { IssuerName string `json:"issuerName,omitempty"` - // Type indicate the issuer is ClusterIssuer or NamespaceIssuer + // Type indicate the issuer is ClusterIssuer or Issuer(namespace issuer), by default, it's Issuer + // +kubebuilder:default:=Issuer Type IssuerType `json:"type,omitempty"` } @@ -69,27 +81,30 @@ const ( NamespaceIssuer IssuerType = "Issuer" ) -// Route will automatically discover podTemplate for Port and SelectLabels if they are not set. -// If Port and SelectLabels are already set, discovery won't work. -// If Port is not set, the first port discovered will be set. -// If SelectLabels are not set, all selectorLabels discovered will be set. +// Route will automatically discover podSpec and label for BackendService. +// If BackendService is already set, discovery won't work. +// If BackendService is not set, the discovery mechanism will work. type Backend struct { - // Protocol means backend-protocol, HTTP, HTTPS, GRPC, GRPCS, AJP and FCGI, By default uses HTTP - Protocol string `json:"protocol,omitempty"` // ReadTimeout used for setting read timeout duration for backend service, the unit is second. ReadTimeout int `json:"readTimeout,omitempty"` // SendTimeout used for setting send timeout duration for backend service, the unit is second. SendTimeout int `json:"sendTimeout,omitempty"` - // Port points to backend service port. - Port intstr.IntOrString `json:"port,omitempty"` - // SelectLabels for backend service. - SelectLabels map[string]string `json:"selectLabels,omitempty"` + // BackendService specifies the backend K8s service and port, it's optional + BackendService *BackendServiceRef `json:"backendService,omitempty"` +} + +// BackendServiceRef specifies the backend K8s service and port, if specified, the two fields are all required +type BackendServiceRef struct { + // Port allow you direct specify backend service port. + Port intstr.IntOrString `json:"port"` + // ServiceName allow you direct specify K8s service for backend service. + ServiceName string `json:"serviceName"` } // RouteStatus defines the observed state of Route type RouteStatus struct { - Ingress *runtimev1alpha1.TypedReference `json:"ingress,omitempty"` - Service *runtimev1alpha1.TypedReference `json:"service,omitempty"` + Ingresses []runtimev1alpha1.TypedReference `json:"ingresses,omitempty"` + Service *runtimev1alpha1.TypedReference `json:"service,omitempty"` runtimev1alpha1.ConditionedStatus `json:",inline"` } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 75a461e17..28f097e1a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,20 +22,16 @@ package v1alpha1 import ( corev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1" - "k8s.io/api/networking/v1beta1" runtime "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Backend) DeepCopyInto(out *Backend) { *out = *in - out.Port = in.Port - if in.SelectLabels != nil { - in, out := &in.SelectLabels, &out.SelectLabels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + if in.BackendService != nil { + in, out := &in.BackendService, &out.BackendService + *out = new(BackendServiceRef) + **out = **in } } @@ -49,6 +45,22 @@ func (in *Backend) DeepCopy() *Backend { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackendServiceRef) DeepCopyInto(out *BackendServiceRef) { + *out = *in + out.Port = in.Port +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendServiceRef. +func (in *BackendServiceRef) DeepCopy() *BackendServiceRef { + if in == nil { + return nil + } + out := new(BackendServiceRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsTrait) DeepCopyInto(out *MetricsTrait) { *out = *in @@ -315,23 +327,13 @@ func (in *RouteSpec) DeepCopyInto(out *RouteSpec) { *out = new(TLS) **out = **in } - if in.DefaultBackend != nil { - in, out := &in.DefaultBackend, &out.DefaultBackend - *out = new(v1beta1.IngressBackend) - (*in).DeepCopyInto(*out) - } - if in.CustomHeaders != nil { - in, out := &in.CustomHeaders, &out.CustomHeaders - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val + if in.Rules != nil { + in, out := &in.Rules, &out.Rules + *out = make([]Rule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Backend != nil { - in, out := &in.Backend, &out.Backend - *out = new(Backend) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteSpec. @@ -347,10 +349,10 @@ func (in *RouteSpec) DeepCopy() *RouteSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RouteStatus) DeepCopyInto(out *RouteStatus) { *out = *in - if in.Ingress != nil { - in, out := &in.Ingress, &out.Ingress - *out = new(corev1alpha1.TypedReference) - **out = **in + if in.Ingresses != nil { + in, out := &in.Ingresses, &out.Ingresses + *out = make([]corev1alpha1.TypedReference, len(*in)) + copy(*out, *in) } if in.Service != nil { in, out := &in.Service, &out.Service @@ -370,6 +372,38 @@ func (in *RouteStatus) DeepCopy() *RouteStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Rule) DeepCopyInto(out *Rule) { + *out = *in + if in.CustomHeaders != nil { + in, out := &in.CustomHeaders, &out.CustomHeaders + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.DefaultBackend != nil { + in, out := &in.DefaultBackend, &out.DefaultBackend + *out = new(corev1alpha1.TypedReference) + **out = **in + } + if in.Backend != nil { + in, out := &in.Backend, &out.Backend + *out = new(Backend) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Rule. +func (in *Rule) DeepCopy() *Rule { + if in == nil { + return nil + } + out := new(Rule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScapeServiceEndPoint) DeepCopyInto(out *ScapeServiceEndPoint) { *out = *in diff --git a/charts/vela-core/crds/core.oam.dev_applicationconfigurations.yaml b/charts/vela-core/crds/core.oam.dev_applicationconfigurations.yaml index f4c65b92d..98dc1f6b7 100644 --- a/charts/vela-core/crds/core.oam.dev_applicationconfigurations.yaml +++ b/charts/vela-core/crds/core.oam.dev_applicationconfigurations.yaml @@ -547,8 +547,6 @@ spec: type: object type: object type: array - required: - - historyWorkloads type: object type: object served: true diff --git a/charts/vela-core/crds/standard.oam.dev_routes.yaml b/charts/vela-core/crds/standard.oam.dev_routes.yaml index 7bc37207d..2b50d500f 100644 --- a/charts/vela-core/crds/standard.oam.dev_routes.yaml +++ b/charts/vela-core/crds/standard.oam.dev_routes.yaml @@ -38,84 +38,90 @@ spec: spec: description: RouteSpec defines the desired state of Route properties: - backend: - description: Backend indicate how to connect backend service If it's - nil, will auto discovery - properties: - port: - anyOf: - - type: integer - - type: string - description: Port points to backend service port. - x-kubernetes-int-or-string: true - protocol: - description: Protocol means backend-protocol, HTTP, HTTPS, GRPC, - GRPCS, AJP and FCGI, By default uses HTTP - type: string - readTimeout: - description: ReadTimeout used for setting read timeout duration - for backend service, the unit is second. - type: integer - selectLabels: - additionalProperties: - type: string - description: SelectLabels for backend service. - type: object - sendTimeout: - description: SendTimeout used for setting send timeout duration - for backend service, the unit is second. - type: integer - type: object - customHeaders: - additionalProperties: - type: string - description: CustomHeaders pass a custom list of headers to the backend - service. - type: object - defaultBackend: - description: DefaultBackend uses serviceName - properties: - resource: - description: Resource is an ObjectRef to another Kubernetes resource - in the namespace of the Ingress object. If resource is specified, - serviceName and servicePort must not be specified. - properties: - apiGroup: - description: APIGroup is the group for the resource being - referenced. If APIGroup is not specified, the specified - Kind must be in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource being referenced - type: string - name: - description: Name is the name of resource being referenced - type: string - required: - - kind - - name - type: object - serviceName: - description: Specifies the name of the referenced service. - type: string - servicePort: - anyOf: - - type: integer - - type: string - description: Specifies the port of the referenced service. - x-kubernetes-int-or-string: true - type: object host: description: Host is the host of the route type: string - path: - description: Path is location Path, default for "/" - type: string - rewriteTarget: - description: RewriteTarget will rewrite request from Path to RewriteTarget - path. + provider: + description: Provider indicate which ingress controller implementation + the route trait will use, by default it's nginx-ingress type: string + rules: + description: Rules contain multiple rules of route + items: + description: Rule defines to route rule + properties: + backend: + description: Backend indicate how to connect backend service + If it's nil, will auto discovery + properties: + backendService: + description: BackendService specifies the backend K8s service + and port, it's optional + properties: + port: + anyOf: + - type: integer + - type: string + description: Port allow you direct specify backend service + port. + x-kubernetes-int-or-string: true + serviceName: + description: ServiceName allow you direct specify K8s + service for backend service. + type: string + required: + - port + - serviceName + type: object + readTimeout: + description: ReadTimeout used for setting read timeout duration + for backend service, the unit is second. + type: integer + sendTimeout: + description: SendTimeout used for setting send timeout duration + for backend service, the unit is second. + type: integer + type: object + customHeaders: + additionalProperties: + type: string + description: CustomHeaders pass a custom list of headers to + the backend service. + type: object + defaultBackend: + description: DefaultBackend will become the ingress default + backend if the backend is not available + properties: + apiVersion: + description: APIVersion of the referenced object. + type: string + kind: + description: Kind of the referenced object. + type: string + name: + description: Name of the referenced object. + type: string + uid: + description: UID of the referenced object. + type: string + required: + - apiVersion + - kind + - name + type: object + name: + description: Name will become the suffix of underlying ingress + created by this rule, if not, will use index as suffix. + type: string + path: + description: Path is location Path, default for "/" + type: string + rewriteTarget: + description: RewriteTarget will rewrite request from Path to + RewriteTarget path. + type: string + type: object + type: array tls: description: TLS indicate route trait will create SSL secret using cert-manager with specified issuer If this is nil, route trait will @@ -124,7 +130,9 @@ spec: issuerName: type: string type: - description: Type indicate the issuer is ClusterIssuer or NamespaceIssuer + default: Issuer + description: Type indicate the issuer is ClusterIssuer or Issuer(namespace + issuer), by default, it's Issuer type: string type: object workloadRef: @@ -187,28 +195,30 @@ spec: - type type: object type: array - ingress: - description: A TypedReference refers to an object by Name, Kind, and - APIVersion. It is commonly used to reference cluster-scoped objects - or objects where the namespace is already known. - properties: - apiVersion: - description: APIVersion of the referenced object. - type: string - kind: - description: Kind of the referenced object. - type: string - name: - description: Name of the referenced object. - type: string - uid: - description: UID of the referenced object. - type: string - required: - - apiVersion - - kind - - name - type: object + ingresses: + items: + description: A TypedReference refers to an object by Name, Kind, + and APIVersion. It is commonly used to reference cluster-scoped + objects or objects where the namespace is already known. + properties: + apiVersion: + description: APIVersion of the referenced object. + type: string + kind: + description: Kind of the referenced object. + type: string + name: + description: Name of the referenced object. + type: string + uid: + description: UID of the referenced object. + type: string + required: + - apiVersion + - kind + - name + type: object + type: array service: description: A TypedReference refers to an object by Name, Kind, and APIVersion. It is commonly used to reference cluster-scoped objects diff --git a/charts/vela-core/templates/definitions/manualscalars.yaml b/charts/vela-core/templates/definitions/manualscalars.yaml index ae9e6eb31..c32a905c2 100644 --- a/charts/vela-core/templates/definitions/manualscalars.yaml +++ b/charts/vela-core/templates/definitions/manualscalars.yaml @@ -5,7 +5,7 @@ metadata: definition.oam.dev/apiVersion: core.oam.dev/v1alpha2 definition.oam.dev/kind: ManualScalerTrait definition.oam.dev/description: "Scale replica for workload" - name: manualscalertraits.core.oam.dev + name: scaler namespace: default spec: appliesToWorkloads: diff --git a/charts/vela-core/templates/definitions/metricstraits.yaml b/charts/vela-core/templates/definitions/metricstraits.yaml index 6721c315e..cf1e094ff 100644 --- a/charts/vela-core/templates/definitions/metricstraits.yaml +++ b/charts/vela-core/templates/definitions/metricstraits.yaml @@ -10,7 +10,7 @@ metadata: apiVersion: core.oam.dev/v1alpha2 kind: TraitDefinition metadata: - name: metricstraits.standard.oam.dev + name: metric namespace: default annotations: definition.oam.dev/apiVersion: standard.oam.dev/v1alpha1 diff --git a/charts/vela-core/templates/definitions/routetrait.yaml b/charts/vela-core/templates/definitions/routetrait.yaml index b85ab8288..22abf7eff 100644 --- a/charts/vela-core/templates/definitions/routetrait.yaml +++ b/charts/vela-core/templates/definitions/routetrait.yaml @@ -1,7 +1,7 @@ apiVersion: core.oam.dev/v1alpha2 kind: TraitDefinition metadata: - name: routes.standard.oam.dev + name: route annotations: definition.oam.dev/apiVersion: standard.oam.dev/v1alpha1 definition.oam.dev/kind: Route diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ab583dbd8..9567726f9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -74,23 +74,3 @@ rules: - get - patch - update -- apiGroups: - - standard.oam.dev - resources: - - routes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - standard.oam.dev - resources: - - routes/status - verbs: - - get - - patch - - update diff --git a/design/route.md b/design/route.md new file mode 100644 index 000000000..eb9f1704b --- /dev/null +++ b/design/route.md @@ -0,0 +1,126 @@ +# Route Trait Design + +The main idea of route trait is to let users have an entrypoint to visit their App. + +In k8s world, if you want to do so, you have to understand K8s [Serivce](https://kubernetes.io/docs/concepts/services-networking/service/) +, [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/), [Ingress Controllers](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/). +It's not easy to get all of these things work well. + +The route trait will help you setup service, ingress automatically according your workload, along with the mTLS enabled. + +The schema is also clean and easy to understand. + +```go + +// RouteSpec defines the desired state of Route +type RouteSpec struct { + // WorkloadReference to the workload whose metrics needs to be exposed + WorkloadReference runtimev1alpha1.TypedReference `json:"workloadRef,omitempty"` + + // Host is the host of the route + Host string `json:"host"` + + // TLS indicate route trait will create SSL secret using cert-manager with specified issuer + // If this is nil, route trait will use a selfsigned issuer + TLS *TLS `json:"tls,omitempty"` + + // Rules contain multiple rules of route + Rules []Rule `json:"rules"` + + // Provider indicate which ingress controller implementation the route trait will use, by default it's nginx-ingress + Provider string `json:"provider,omitempty"` +} + +// Rule defines to route rule +type Rule struct { + // Path is location Path, default for "/" + Path string `json:"path,omitempty"` + + + // RewriteTarget will rewrite request from Path to RewriteTarget path. + RewriteTarget string `json:"rewriteTarget,omitempty"` + + // CustomHeaders pass a custom list of headers to the backend service. + CustomHeaders map[string]string `json:"customHeaders,omitempty"` + + // DefaultBackend will become the ingress default backend if the backend is not available + DefaultBackend runtimev1alpha1.TypedReference `json:"defaultBackend,omitempty"` + + // Backend indicate how to connect backend service + // If it's nil, will auto discovery + Backend *Backend `json:"backend,omitempty"` +} + +type TLS struct { + IssuerName string `json:"issuerName,omitempty"` + + // Type indicate the issuer is ClusterIssuer or NamespaceIssuer + Type IssuerType `json:"type,omitempty"` +} + +type IssuerType string + +const ( + ClusterIssuer IssuerType = "ClusterIssuer" + NamespaceIssuer IssuerType = "Issuer" +) + +// Route will automatically discover podSpec and label for BackendService. +// If BackendService is already set, discovery won't work. +// If BackendService is not set, the discovery mechanism will work. +type Backend struct { + // ReadTimeout used for setting read timeout duration for backend service, the unit is second. + ReadTimeout int `json:"readTimeout,omitempty"` + // SendTimeout used for setting send timeout duration for backend service, the unit is second. + SendTimeout int `json:"sendTimeout,omitempty"` + // BackendService specifies the backend K8s service and port + BackendService *BackendServiceRef `json:"backendService,omitempty"` +} + +type BackendServiceRef struct { + // Port allow you direct specify backend service port. + Port intstr.IntOrString `json:"port,omitempty"` + // ServiceName allow you direct specify K8s service for backend service. + ServiceName string `json:"serviceName,omitempty"` +} +``` + +Route Trait specifies a target workload by using `workloadRef`, in OAM system, this field will be filled automatically +by OAM runtime. + +Besides `workloadRef`, one Route will have only one `host` and many rules. `host` is actually your app's visiting URL. +It's required and will be used to generate mTLS secrets. + +Route Trait designed to be compatible with different ingress controller implementations, the `provider` field will allow +you to give a specified ingress controller type. Currently, only nginx-ingress is supported. + +The `tls` field allow you to specify a TLS for this route with an IssuerName, the IssuerName pointing to an Issuer Object +created by cert-manager. Cert-manager and ingress controller will handle certificate creation and binding. +If not specified, a selfsigned issuer will be created. + +If no rule specified, route trait will create one rule automatically and match with the port. + +For every rule, we will create an ingress. In the rule, you could specify `path`, `rewriteTarget`, `customHeaders` +and `defaultBackend`. All rules will using the same `tls`, `host` and `provider`. + +`defaultBackend` will become the ingress default backend with K8s Object(apiVersion/kind/name). + +`backend` of the rule is completely optional. + +If backendService is specified, it will use it as backend of this rule. If not, +the route trait can automatically discovery backend settings from workload. + +## Discovery mechanism + +1. Check ChildResource of the workload, if there already has an existing K8s service match the Backend port, use it. +2. If there's no k8s service, this means we need to create one. In order to create K8s service, we need two information +`Container Port` and `Pod SelectorLabels`. + - 2.1 Use [`PodSpecable` mechanism](https://github.com/crossplane/oam-kubernetes-runtime/blob/master/design/one-pager-podspecable-workload.md), +route trait will check `WorkloadDefinition` for podSpec field, with the `podSpec` field, we can easily find the container port. + * If podSpecPath` is specified, we will use the workload labels as `Pod SelectorLabels`. + * If `workload.oam.dev/podspecable: true` but no `podSpecPath`, will use `spec.Template` as `PodTemplate`, which means + we can get `Pod SelectorLabels` from `spec.Template.Labels`. + - 2.2 Use ChildResource: If No `PodSpecable` mechanism found in workload, we will continue discovery child resources of workload. If there + is a valid `PodTemplate` structure in child resource, we will regard it as discovery target, use the same strategy like + `workload.oam.dev/podspecable: true` but no `podSpecPath`. + diff --git a/documentation/samples/route-demo/definition.yaml b/documentation/samples/route-demo/definition.yaml new file mode 100644 index 000000000..c1dd83832 --- /dev/null +++ b/documentation/samples/route-demo/definition.yaml @@ -0,0 +1,53 @@ +apiVersion: core.oam.dev/v1alpha2 +kind: TraitDefinition +metadata: + name: route + annotations: + definition.oam.dev/apiVersion: standard.oam.dev/v1alpha1 + definition.oam.dev/kind: Route + definition.oam.dev/description: "Add a route for workload" +spec: + appliesToWorkloads: + - core.oam.dev/v1alpha2.ContainerizedWorkload + - standard.oam.dev/v1alpha1.PodSpecWorkload + - deployments.apps + - webservice + workloadRefPath: spec.workloadRef + definitionRef: + name: routes.standard.oam.dev +--- +apiVersion: core.oam.dev/v1alpha2 +kind: WorkloadDefinition +metadata: + name: webservice + annotations: + definition.oam.dev/apiVersion: "standard.oam.dev/v1alpha1" + definition.oam.dev/kind: "PodSpecWorkload" + definition.oam.dev/description: "Long running service with ports exposed" +spec: + definitionRef: + name: podspecworkloads.standard.oam.dev + childResourceKinds: + - apiVersion: apps/v1 + kind: Deployment + - apiVersion: v1 + kind: Service +--- +apiVersion: core.oam.dev/v1alpha2 +kind: WorkloadDefinition +metadata: + name: deployment + labels: + workload.oam.dev/podspecable: "true" +spec: + definitionRef: + name: deployments.apps +--- +apiVersion: core.oam.dev/v1alpha2 +kind: WorkloadDefinition +metadata: + name: deploy +spec: + podSpecPath: spec.template.spec + definitionRef: + name: deployments.apps \ No newline at end of file diff --git a/documentation/samples/route-demo/deploy.yaml b/documentation/samples/route-demo/deploy.yaml new file mode 100644 index 000000000..3d64b6c02 --- /dev/null +++ b/documentation/samples/route-demo/deploy.yaml @@ -0,0 +1,60 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: oam-env-default +spec: + selfSigned: {} + +--- +apiVersion: core.oam.dev/v1alpha2 +kind: Component +metadata: + name: mycomp +spec: + workload: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: mycomp + labels: + workload.oam.dev/type: deploy + component.oam.dev/name: mycomp + spec: + replicas: 1 + selector: + matchLabels: + component.oam.dev/name: mycomp + template: + metadata: + labels: + component.oam.dev/name: mycomp + spec: + containers: + - image: crccheck/hello-world + imagePullPolicy: Always + name: mycomp + ports: + - containerPort: 8000 + name: default + protocol: TCP +--- +apiVersion: core.oam.dev/v1alpha2 +kind: ApplicationConfiguration +metadata: + name: myapp +spec: + components: + - componentName: mycomp + traits: + - trait: + apiVersion: standard.oam.dev/v1alpha1 + kind: Route + metadata: + labels: + trait.oam.dev/type: route + spec: + host: mycomp.mytest.com +# backend: +# port: 8000 + tls: + issuerName: oam-env-default \ No newline at end of file diff --git a/documentation/samples/route-demo/deployment-app.yaml b/documentation/samples/route-demo/deployment-app.yaml new file mode 100644 index 000000000..e2db81a8c --- /dev/null +++ b/documentation/samples/route-demo/deployment-app.yaml @@ -0,0 +1,58 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: oam-env-default +spec: + selfSigned: {} + +--- +apiVersion: core.oam.dev/v1alpha2 +kind: Component +metadata: + name: mycomp +spec: + workload: + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + workload.oam.dev/type: webservice + spec: + replicas: 1 + selector: + matchLabels: + component.oam.dev/name: mycomp + template: + metadata: + labels: + component.oam.dev/name: mycomp + spec: + containers: + - image: crccheck/hello-world + imagePullPolicy: Always + name: mycomp + ports: + - containerPort: 8000 + name: default + protocol: TCP +--- +apiVersion: core.oam.dev/v1alpha2 +kind: ApplicationConfiguration +metadata: + name: myapp +spec: + components: + - componentName: mycomp + traits: + - trait: + apiVersion: standard.oam.dev/v1alpha1 + kind: Route + metadata: + labels: + trait.oam.dev/type: route + spec: + host: mycomp.mytest.com + tls: + issuerName: oam-env-default + + diff --git a/documentation/samples/route-demo/webservice-app.yaml b/documentation/samples/route-demo/webservice-app.yaml new file mode 100644 index 000000000..ebe24806f --- /dev/null +++ b/documentation/samples/route-demo/webservice-app.yaml @@ -0,0 +1,51 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: oam-env-default +spec: + selfSigned: {} + +--- +apiVersion: core.oam.dev/v1alpha2 +kind: Component +metadata: + name: mycomp +spec: + workload: + apiVersion: standard.oam.dev/v1alpha1 + kind: PodSpecWorkload + metadata: + name: mycomp + labels: + workload.oam.dev/type: webservice + spec: + podSpec: + containers: + - image: crccheck/hello-world + name: mycomp + ports: + - containerPort: 8000 + name: default + protocol: TCP + replicas: 1 +--- +apiVersion: core.oam.dev/v1alpha2 +kind: ApplicationConfiguration +metadata: + name: myapp +spec: + components: + - componentName: mycomp + traits: + - trait: + apiVersion: standard.oam.dev/v1alpha1 + kind: Route + metadata: + labels: + trait.oam.dev/type: route + spec: + host: mycomp.mytest.com +# backend: +# port: 8000 + tls: + issuerName: oam-env-default \ No newline at end of file diff --git a/go.mod b/go.mod index 6ad19889a..db58ec3d4 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/briandowns/spinner v1.11.1 github.com/coreos/prometheus-operator v0.41.1 github.com/crossplane/crossplane-runtime v0.9.0 - github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1 + github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1.0.20201015120208-c65ccab4f9c1 github.com/fatih/color v1.9.0 github.com/gertd/go-pluralize v0.1.7 github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index f4d8c2bc8..a05309f00 100644 --- a/go.sum +++ b/go.sum @@ -281,6 +281,8 @@ github.com/crossplane/crossplane-runtime v0.9.0/go.mod h1:gNY/21MLBaz5KNP7hmfXbB github.com/crossplane/crossplane-tools v0.0.0-20200219001116-bb8b2ce46330/go.mod h1:C735A9X0x0lR8iGVOOxb49Mt70Ua4EM2b7PGaRPBLd4= github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1 h1:N9999ECMaJYf+yhiNgAXk2HIgMeT9E5M7Y1AcXut8dU= github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1/go.mod h1:D+MDS5vrJZWEA5cxr5kyzCSRQwrt1hLD3ONgC7sVMmc= +github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1.0.20201015120208-c65ccab4f9c1 h1:uAnGnossZ4biy97EKG6xj9zcGD94Uo1kwbdEPR9lZag= +github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1.0.20201015120208-c65ccab4f9c1/go.mod h1:D+MDS5vrJZWEA5cxr5kyzCSRQwrt1hLD3ONgC7sVMmc= github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/daixiang0/gci v0.0.0-20200727065011-66f1df783cb2/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4= diff --git a/pkg/application/app.go b/pkg/application/app.go index 17f5855d3..92a91da1d 100644 --- a/pkg/application/app.go +++ b/pkg/application/app.go @@ -327,7 +327,7 @@ func (app *Application) GetComponentTraits(componentName string, env *types.EnvM return nil, err } //TODO(wonderflow): handle trait data input/output here - obj.SetAnnotations(map[string]string{types.AnnTraitDef: traitType}) + obj.SetLabels(map[string]string{oam.TraitTypeLabel: traitType}) traits = append(traits, v1alpha2.ComponentTrait{Trait: runtime.RawExtension{Object: obj}}) } return traits, nil @@ -369,13 +369,13 @@ func (app *Application) OAM(env *types.EnvMeta) ([]v1alpha2.Component, v1alpha2. if err != nil { return nil, v1alpha2.ApplicationConfiguration{}, nil, err } - anns := component.Annotations - if anns == nil { - anns = map[string]string{types.AnnWorkloadDef: workloadType} + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{oam.WorkloadTypeLabel: workloadType} } else { - anns[types.AnnWorkloadDef] = workloadType + labels[oam.WorkloadTypeLabel] = workloadType } - component.Annotations = anns + obj.SetLabels(labels) component.Spec.Workload.Object = obj components = append(components, component) diff --git a/pkg/commands/ls.go b/pkg/commands/ls.go index bbed15116..53c9d4b99 100644 --- a/pkg/commands/ls.go +++ b/pkg/commands/ls.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" "sigs.k8s.io/controller-runtime/pkg/client" + runtimeoam "github.com/crossplane/oam-kubernetes-runtime/pkg/oam" "github.com/oam-dev/kubevela/api/types" "github.com/oam-dev/kubevela/pkg/application" cmdutil "github.com/oam-dev/kubevela/pkg/commands/util" @@ -132,7 +133,7 @@ func mergeStagingComponents(deployed []apis.ComponentMeta, env *types.EnvMeta, i all = append(all, apis.ComponentMeta{ Name: c.Name, App: app.Name, - WorkloadName: c.Annotations[types.AnnWorkloadDef], + WorkloadName: c.Labels[runtimeoam.WorkloadTypeLabel], TraitNames: traits, Status: types.StatusStaging, CreatedTime: app.CreateTime.String(), diff --git a/pkg/controller/v1alpha1/routes/ingress/api.go b/pkg/controller/v1alpha1/routes/ingress/api.go new file mode 100644 index 000000000..22feca35c --- /dev/null +++ b/pkg/controller/v1alpha1/routes/ingress/api.go @@ -0,0 +1,26 @@ +package ingress + +import ( + "fmt" + + "k8s.io/api/networking/v1beta1" + + standardv1alpha1 "github.com/oam-dev/kubevela/api/v1alpha1" +) + +const TypeNginx = "nginx" + +type RouteIngress interface { + Construct(routeTrait *standardv1alpha1.Route) []*v1beta1.Ingress +} + +func GetRouteIngress(provider string) (RouteIngress, error) { + var routeIngress RouteIngress + switch provider { + case TypeNginx, "": + routeIngress = &Nginx{} + default: + return nil, fmt.Errorf("unknow route ingress provider '%v', only '%s' is supported now", provider, TypeNginx) + } + return routeIngress, nil +} diff --git a/pkg/controller/v1alpha1/routes/ingress/api_test.go b/pkg/controller/v1alpha1/routes/ingress/api_test.go new file mode 100644 index 000000000..795bd8b58 --- /dev/null +++ b/pkg/controller/v1alpha1/routes/ingress/api_test.go @@ -0,0 +1,16 @@ +package ingress + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetRouteIngress(t *testing.T) { + _, err := GetRouteIngress("nginx") + assert.NoError(t, err) + _, err = GetRouteIngress("") + assert.NoError(t, err) + _, err = GetRouteIngress("istio") + assert.EqualError(t, err, "unknow route ingress provider 'istio', only 'nginx' is supported now") +} diff --git a/pkg/controller/v1alpha1/routes/ingress/nginx_ingress.go b/pkg/controller/v1alpha1/routes/ingress/nginx_ingress.go new file mode 100644 index 000000000..2d79ce71b --- /dev/null +++ b/pkg/controller/v1alpha1/routes/ingress/nginx_ingress.go @@ -0,0 +1,124 @@ +package ingress + +import ( + "fmt" + "reflect" + "strconv" + + standardv1alpha1 "github.com/oam-dev/kubevela/api/v1alpha1" + + v1 "k8s.io/api/core/v1" + "k8s.io/api/networking/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" +) + +type Nginx struct{} + +var _ RouteIngress = &Nginx{} + +func (*Nginx) Construct(routeTrait *standardv1alpha1.Route) []*v1beta1.Ingress { + + var ingresses []*v1beta1.Ingress + for idx, rule := range routeTrait.Spec.Rules { + name := rule.Name + if name == "" { + name = strconv.Itoa(idx) + } + backend := rule.Backend + if backend == nil || backend.BackendService == nil { + continue + } + + var annotations = make(map[string]string) + + annotations["kubernetes.io/ingress.class"] = TypeNginx + + // SSL + var issuerAnn = "cert-manager.io/issuer" + if routeTrait.Spec.TLS.Type == standardv1alpha1.ClusterIssuer { + issuerAnn = "cert-manager.io/cluster-issuer" + } + annotations[issuerAnn] = routeTrait.Spec.TLS.IssuerName + + // Rewrite + if rule.RewriteTarget != "" { + annotations["ingress.kubernetes.io/rewrite-target"] = rule.RewriteTarget + } + + // Custom headers + var headerSnippet string + for k, v := range rule.CustomHeaders { + headerSnippet += fmt.Sprintf("more_set_headers \"%s: %s\";\n", k, v) + } + if headerSnippet != "" { + annotations["nginx.ingress.kubernetes.io/configuration-snippet"] = headerSnippet + } + + //Send timeout + if backend.SendTimeout != 0 { + annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = strconv.Itoa(backend.SendTimeout) + } + + //Read timeout + if backend.ReadTimeout != 0 { + annotations["nginx.ingress.kubernetes.io/proxy‑read‑timeout"] = strconv.Itoa(backend.ReadTimeout) + } + + ingress := &v1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{ + Kind: reflect.TypeOf(v1beta1.Ingress{}).Name(), + APIVersion: v1beta1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: routeTrait.Name + "-" + name, + Namespace: routeTrait.Namespace, + Annotations: annotations, + Labels: routeTrait.GetLabels(), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: routeTrait.GetObjectKind().GroupVersionKind().GroupVersion().String(), + Kind: routeTrait.GetObjectKind().GroupVersionKind().Kind, + UID: routeTrait.GetUID(), + Name: routeTrait.GetName(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + }, + } + ingress.Spec.TLS = []v1beta1.IngressTLS{ + { + Hosts: []string{routeTrait.Spec.Host}, + SecretName: routeTrait.Name + "-" + name + "-cert", + }, + } + if rule.DefaultBackend != nil { + ingress.Spec.Backend = &v1beta1.IngressBackend{ + Resource: &v1.TypedLocalObjectReference{ + APIGroup: &rule.DefaultBackend.APIVersion, + Kind: rule.DefaultBackend.Kind, + Name: rule.DefaultBackend.Name, + }, + } + } + ingress.Spec.Rules = []v1beta1.IngressRule{ + { + Host: routeTrait.Spec.Host, + IngressRuleValue: v1beta1.IngressRuleValue{HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: rule.Path, + Backend: v1beta1.IngressBackend{ + ServiceName: backend.BackendService.ServiceName, + ServicePort: backend.BackendService.Port, + }, + }, + }, + }}, + }, + } + ingresses = append(ingresses, ingress) + } + return ingresses +} diff --git a/pkg/controller/v1alpha1/routes/ingress/nginx_ingress_test.go b/pkg/controller/v1alpha1/routes/ingress/nginx_ingress_test.go new file mode 100644 index 000000000..35d995686 --- /dev/null +++ b/pkg/controller/v1alpha1/routes/ingress/nginx_ingress_test.go @@ -0,0 +1,109 @@ +package ingress + +import ( + "strconv" + "testing" + + "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/api/networking/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + + standardv1alpha1 "github.com/oam-dev/kubevela/api/v1alpha1" +) + +func TestConstruct(t *testing.T) { + tests := map[string]struct { + routeTrait *standardv1alpha1.Route + exp []*v1beta1.Ingress + }{ + "normal case": { + routeTrait: &standardv1alpha1.Route{ + TypeMeta: metav1.TypeMeta{ + Kind: "Route", + APIVersion: "standard.oam.dev/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "trait-test", + }, + Spec: standardv1alpha1.RouteSpec{ + Host: "test.abc", + TLS: &standardv1alpha1.TLS{ + IssuerName: "test-issuer", + Type: "Issuer", + }, + Rules: []standardv1alpha1.Rule{ + { + Name: "myrule1", + Backend: &standardv1alpha1.Backend{BackendService: &standardv1alpha1.BackendServiceRef{ServiceName: "test", Port: intstr.FromInt(3030)}}, + DefaultBackend: &v1alpha1.TypedReference{ + APIVersion: "k8s.example.com/v1", + Kind: "StorageBucket", + Name: "static-assets", + }, + }, + }, + }, + }, + exp: []*v1beta1.Ingress{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Ingress", + APIVersion: "networking.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "trait-test-myrule1", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + "cert-manager.io/issuer": "test-issuer", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "standard.oam.dev/v1alpha1", + Kind: "Route", + Name: "trait-test", + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + }, + Spec: v1beta1.IngressSpec{ + TLS: []v1beta1.IngressTLS{ + { + Hosts: []string{"test.abc"}, + SecretName: "trait-test-myrule1-cert", + }, + }, + Backend: &v1beta1.IngressBackend{Resource: &v1.TypedLocalObjectReference{ + APIGroup: pointer.StringPtr("k8s.example.com/v1"), + Kind: "StorageBucket", + Name: "static-assets", + }}, + Rules: []v1beta1.IngressRule{ + { + Host: "test.abc", + IngressRuleValue: v1beta1.IngressRuleValue{HTTP: &v1beta1.HTTPIngressRuleValue{Paths: []v1beta1.HTTPIngressPath{ + { + Path: "", + Backend: v1beta1.IngressBackend{ServiceName: "test", ServicePort: intstr.FromInt(3030)}, + }, + }}}, + }, + }, + }, + }, + }, + }, + } + for message, ti := range tests { + nginx := &Nginx{} + got := nginx.Construct(ti.routeTrait) + assert.Equal(t, len(ti.exp), len(got)) + for idx := range ti.exp { + assert.Equal(t, ti.exp[idx], got[idx], message+" index "+strconv.Itoa(idx)) + } + } +} diff --git a/pkg/controller/v1alpha1/routes/route_controller.go b/pkg/controller/v1alpha1/routes/route_controller.go index 18f47dc4c..b57a4a2f9 100644 --- a/pkg/controller/v1alpha1/routes/route_controller.go +++ b/pkg/controller/v1alpha1/routes/route_controller.go @@ -21,11 +21,11 @@ import ( "encoding/json" "fmt" "reflect" - "strconv" "github.com/oam-dev/kubevela/api/v1alpha1" standardv1alpha1 "github.com/oam-dev/kubevela/api/v1alpha1" "github.com/oam-dev/kubevela/pkg/controller/common" + "github.com/oam-dev/kubevela/pkg/controller/v1alpha1/routes/ingress" cpv1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1" runtimev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1" @@ -51,15 +51,6 @@ const ( errCreateIssuer = "failed to create cert-manager Issuer" ) -var ( - // oamServiceLabel is the pre-defined labels for any serviceMonitor - // created by the RouteTrait - oamServiceLabel = map[string]string{ - "k8s-app": "oam", - "controller": "routeTrait", - } -) - // Reconciler reconciles a Route object type Reconciler struct { client.Client @@ -70,7 +61,6 @@ type Reconciler struct { // +kubebuilder:rbac:groups=standard.oam.dev,resources=routes,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=standard.oam.dev,resources=routes/status,verbs=get;update;patch - func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ctx := context.Background() mLog := r.Log.WithValues("route", req.NamespacedName) @@ -83,7 +73,6 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { } mLog.Info("Get the route trait", "host", routeTrait.Spec.Host, - "path", routeTrait.Spec.Path, "workload reference", routeTrait.Spec.WorkloadReference, "labels", routeTrait.GetLabels()) @@ -105,35 +94,15 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { oamutil.PatchCondition(ctx, r, &routeTrait, cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrLocatingWorkload))) } - - // try to see if the workload already has services as child resources, and match for our route - svc, exist, svcPort, err := r.fetchService(ctx, mLog, workload, &routeTrait) - if err != nil && !apierrors.IsNotFound(err) { - r.record.Event(eventObj, event.Warning(common.ErrLocatingService, err)) - return oamutil.ReconcileWaitResult, - oamutil.PatchCondition(ctx, r, &routeTrait, - cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrLocatingService))) - } - - // Create Services - if !exist { - // no service found, we will create service according to rule - svc, svcPort, err = r.createService(ctx, mLog, workload, &routeTrait) - if err != nil { - r.record.Event(eventObj, event.Warning(common.ErrCreatingService, err)) - return oamutil.ReconcileWaitResult, - oamutil.PatchCondition(ctx, r, &routeTrait, - cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrCreatingService))) + var svc *runtimev1alpha1.TypedReference + if NeedDiscovery(&routeTrait) { + if svc, err = r.discoveryAndFillBackend(ctx, mLog, eventObj, workload, &routeTrait); err != nil { + return oamutil.ReconcileWaitResult, err } - r.record.Event(eventObj, event.Normal("Service created", - fmt.Sprintf("successfully automatically created a service `%s`", svc.Name))) - } else { - mLog.Info("workload already has service as child resource, will not create new", "workloadName", workload.GetName()) } - // Create Issuers - var issuer standardv1alpha1.TLS - if routeTrait.Spec.TLS == nil || routeTrait.Spec.TLS.IssuerName == "" { + // Create Issuer + if routeTrait.Spec.TLS == nil { issuerName, err := r.createSelfsignedIssuer(ctx, &routeTrait) if err != nil { r.record.Event(eventObj, event.Warning(errCreateIssuer, err)) @@ -143,42 +112,93 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { } r.record.Event(eventObj, event.Normal("Issuer created", fmt.Sprintf("successfully automatically created a Issuer for route TLS `%s`", issuerName))) - issuer.Type = standardv1alpha1.NamespaceIssuer - issuer.IssuerName = issuerName - } else { - issuer = *routeTrait.Spec.TLS + // All rules will use the same selfsigned issuer. + routeTrait.Spec.TLS = &v1alpha1.TLS{ + IssuerName: issuerName, + Type: standardv1alpha1.NamespaceIssuer, + } + } + + ingressConstructer, err := ingress.GetRouteIngress(routeTrait.Spec.Provider) + if err != nil { + mLog.Error(err, "Failed to get routeIngress, use nginx route instead") + ingressConstructer = &ingress.Nginx{} } // Create Ingress // construct the serviceMonitor that hooks the service to the prometheus server - ingress := constructNginxIngress(&routeTrait, issuer, svc, svcPort) + ingresses := ingressConstructer.Construct(&routeTrait) // server side apply the serviceMonitor, only the fields we set are touched applyOpts := []client.PatchOption{client.ForceOwnership, client.FieldOwner(routeTrait.GetUID())} - if err := r.Patch(ctx, ingress, client.Apply, applyOpts...); err != nil { - mLog.Error(err, "Failed to apply to ingress") - r.record.Event(eventObj, event.Warning(errApplyNginxIngress, err)) - return oamutil.ReconcileWaitResult, - oamutil.PatchCondition(ctx, r, &routeTrait, - cpv1alpha1.ReconcileError(errors.Wrap(err, errApplyNginxIngress))) + for _, ingress := range ingresses { + if err := r.Patch(ctx, ingress, client.Apply, applyOpts...); err != nil { + mLog.Error(err, "Failed to apply to ingress") + r.record.Event(eventObj, event.Warning(errApplyNginxIngress, err)) + return oamutil.ReconcileWaitResult, + oamutil.PatchCondition(ctx, r, &routeTrait, + cpv1alpha1.ReconcileError(errors.Wrap(err, errApplyNginxIngress))) + } + r.record.Event(eventObj, event.Normal("Nginx Ingress created", + fmt.Sprintf("successfully server side patched a route trait `%s`", routeTrait.Name))) } - r.record.Event(eventObj, event.Normal("Nginx Ingress created", - fmt.Sprintf("successfully server side patched a route trait `%s`", routeTrait.Name))) - // TODO(wonderflow): GC mechanism for no used ingress, service, issuer - routeTrait.Status.Service = svc - routeTrait.Status.Ingress = &runtimev1alpha1.TypedReference{ - APIVersion: v1beta1.SchemeGroupVersion.String(), - Kind: reflect.TypeOf(v1beta1.Ingress{}).Name(), - Name: ingress.Name, - UID: routeTrait.UID, + var ingressCreated []runtimev1alpha1.TypedReference + for _, ingress := range ingresses { + ingressCreated = append(ingressCreated, runtimev1alpha1.TypedReference{ + APIVersion: v1beta1.SchemeGroupVersion.String(), + Kind: reflect.TypeOf(v1beta1.Ingress{}).Name(), + Name: ingress.Name, + UID: routeTrait.UID, + }) } - return ctrl.Result{}, oamutil.PatchCondition(ctx, r, &routeTrait) + routeTrait.Status.Ingresses = ingressCreated + routeTrait.Status.Service = svc + return ctrl.Result{}, r.Status().Update(ctx, &routeTrait) } -// create a service that targets the exposed workload pod -func (r *Reconciler) createService(ctx context.Context, mLog logr.Logger, workload *unstructured.Unstructured, - routeTrait *v1alpha1.Route) (*runtimev1alpha1.TypedReference, int32, error) { +// discoveryAndFillBackend will automatically discovery backend for route +func (r *Reconciler) discoveryAndFillBackend(ctx context.Context, mLog logr.Logger, eventObj runtime.Object, workload *unstructured.Unstructured, + routeTrait *v1alpha1.Route) (*runtimev1alpha1.TypedReference, error) { + + // Fetch the child childResources list from the corresponding workload + childResources, err := oamutil.FetchWorkloadChildResources(ctx, mLog, r, workload) + if err != nil { + mLog.Error(err, "Error while fetching the workload child childResources", "workload kind", workload.GetKind(), + "workload name", workload.GetName()) + if !apierrors.IsNotFound(err) { + return nil, err + } + } + // try to see if the workload already has services in child childResources, and match for our route + err = r.fillBackendByCheckChildResource(mLog, routeTrait, childResources) + if err != nil && !apierrors.IsNotFound(err) { + r.record.Event(eventObj, event.Warning(common.ErrLocatingService, err)) + return nil, oamutil.PatchCondition(ctx, r, routeTrait, + cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrLocatingService))) + } + + // Check if still need discovery after childResource filled. + if NeedDiscovery(routeTrait) { + // no service found, we will create service according to rule + svc, err := r.fillBackendByCreatedService(ctx, mLog, workload, routeTrait, childResources) + if err != nil { + r.record.Event(eventObj, event.Warning(common.ErrCreatingService, err)) + return nil, oamutil.PatchCondition(ctx, r, routeTrait, + cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrCreatingService))) + } + r.record.Event(eventObj, event.Normal("Service created", + fmt.Sprintf("successfully automatically created a service `%s`", svc.Name))) + return svc, nil + } + mLog.Info("workload already has service as child resource, will not create service", "workloadName", workload.GetName()) + return nil, nil + +} + +// fillBackendByCreatedService will automatically create service by discovery podTemplate or podSpec. +func (r *Reconciler) fillBackendByCreatedService(ctx context.Context, mLog logr.Logger, workload *unstructured.Unstructured, + routeTrait *v1alpha1.Route, childResources []*unstructured.Unstructured) (*runtimev1alpha1.TypedReference, error) { oamService := &corev1.Service{ TypeMeta: metav1.TypeMeta{ @@ -186,9 +206,9 @@ func (r *Reconciler) createService(ctx context.Context, mLog logr.Logger, worklo APIVersion: common.ServiceAPIVersion, }, ObjectMeta: metav1.ObjectMeta{ - Name: "route-" + workload.GetName(), - Namespace: workload.GetNamespace(), - Labels: oamServiceLabel, + Name: routeTrait.GetName(), + Namespace: routeTrait.GetNamespace(), + Labels: filterLabels(routeTrait.GetLabels()), OwnerReferences: []metav1.OwnerReference{ { APIVersion: routeTrait.GetObjectKind().GroupVersionKind().GroupVersion().String(), @@ -204,97 +224,70 @@ func (r *Reconciler) createService(ctx context.Context, mLog logr.Logger, worklo Type: corev1.ServiceTypeClusterIP, }, } - // assign selector - if routeTrait.Spec.Backend != nil && len(routeTrait.Spec.Backend.SelectLabels) != 0 { - oamService.Spec.Selector = routeTrait.Spec.Backend.SelectLabels - } - port, labels, err := DiscoverPortLabel(ctx, mLog, workload, r) - if err == nil { - if len(oamService.Spec.Selector) == 0 { - oamService.Spec.Selector = labels - } - if routeTrait.Spec.Backend == nil { - routeTrait.Spec.Backend = &standardv1alpha1.Backend{Port: port} - } else if routeTrait.Spec.Backend.Port.String() == "0" { - routeTrait.Spec.Backend.Port = port - } - } else { + ports, labels, err := DiscoverPortsLabel(ctx, workload, r, childResources) + if err != nil { mLog.Info("[WARN] fail to discovery port and label", "err", err) + return nil, err } + oamService.Spec.Selector = labels - var servicePort int32 = 443 - oamService.Spec.Ports = []corev1.ServicePort{ - { - Port: servicePort, - TargetPort: routeTrait.Spec.Backend.Port, + // use the same port + for _, port := range ports { + oamService.Spec.Ports = append(oamService.Spec.Ports, corev1.ServicePort{ + Port: int32(port.IntValue()), + TargetPort: port, Protocol: corev1.ProtocolTCP, - }, + }) } // server side apply the service, only the fields we set are touched applyOpts := []client.PatchOption{client.ForceOwnership, client.FieldOwner(routeTrait.GetUID())} if err := r.Patch(ctx, oamService, client.Apply, applyOpts...); err != nil { mLog.Error(err, "Failed to apply to service") - return nil, servicePort, err + return nil, err } + FillRouteTraitWithService(oamService, routeTrait) return &runtimev1alpha1.TypedReference{ APIVersion: common.ServiceAPIVersion, Kind: common.ServiceKind, Name: oamService.Name, UID: routeTrait.UID, - }, servicePort, nil + }, nil } // Assume the workload or it's childResource will always having spec.template as PodTemplate if discoverable -func DiscoverPortLabel(ctx context.Context, mLog logr.Logger, workload *unstructured.Unstructured, r client.Reader) (intstr.IntOrString, map[string]string, error) { - var resources = []*unstructured.Unstructured{workload} - // Fetch the child resources list from the corresponding workload - childResources, err := oamutil.FetchWorkloadChildResources(ctx, mLog, r, workload) - if err == nil { - resources = append(resources, childResources...) - } else { - mLog.Info("[WARN] fail to fetch workload child resource", "name", workload.GetName(), "err", err) +func DiscoverPortsLabel(ctx context.Context, workload *unstructured.Unstructured, r client.Reader, childResources []*unstructured.Unstructured) ([]intstr.IntOrString, map[string]string, error) { + + // here is the logic follows the design https://github.com/crossplane/oam-kubernetes-runtime/blob/master/design/one-pager-podspecable-workload.md#proposal + // Get WorkloadDefinition + workloadDef, err := oamutil.FetchWorkloadDefinition(ctx, r, workload) + if err != nil { + return nil, nil, err } + podSpecPath, ok := GetPodSpecPath(workloadDef) + if podSpecPath != "" { + ports, err := discoveryFromPodSpec(workload, podSpecPath) + if err != nil { + return nil, nil, err + } + return ports, filterLabels(workload.GetLabels()), nil + } + if ok { + return discoveryFromPodTemplate(workload, "spec", "template") + } + + // If workload is not podSpecable, try to detect it's child resource + var resources = []*unstructured.Unstructured{workload} + resources = append(resources, childResources...) var gatherErrs []error for _, w := range resources { - port, labels, err := discoveryFromObject(w) + port, labels, err := discoveryFromPodTemplate(w, "spec", "template") if err == nil { return port, labels, nil } gatherErrs = append(gatherErrs, err) } - return intstr.IntOrString{}, nil, fmt.Errorf("can't discovery port from workload %v %v.%v and it's child resource, errorList: %v", workload.GetName(), workload.GetAPIVersion(), workload.GetKind(), gatherErrs) -} - -func discoveryFromObject(w *unstructured.Unstructured) (intstr.IntOrString, map[string]string, error) { - obj, found, _ := unstructured.NestedMap(w.Object, "spec", "template") - if !found { - return intstr.IntOrString{}, nil, fmt.Errorf("not have spec.template in workload %v", w.GetName()) - } - data, err := json.Marshal(obj) - if err != nil { - return intstr.IntOrString{}, nil, fmt.Errorf("workload %v convert object err %v", w.GetName(), err) - } - var template corev1.PodTemplate - err = json.Unmarshal(data, &template) - if err != nil { - return intstr.IntOrString{}, nil, fmt.Errorf("workload %v convert object to PodTemplate err %v", w.GetName(), err) - } - port := getFirstPort(template.Template.Spec.Containers) - if port == 0 { - return intstr.IntOrString{}, nil, fmt.Errorf("no port found in workload %v", w.GetName()) - } - return intstr.FromInt(int(port)), template.Labels, nil -} - -func getFirstPort(cs []corev1.Container) int32 { - //TODO(wonderflow): exclude some sidecars - for _, container := range cs { - for _, p := range container.Ports { - return p.ContainerPort - } - } - return 0 + return nil, nil, fmt.Errorf("fail to automatically discovery backend from workload %v(%v.%v) and it's child resource, errorList: %v", workload.GetName(), workload.GetAPIVersion(), workload.GetKind(), gatherErrs) } func (r *Reconciler) createSelfsignedIssuer(ctx context.Context, routeTrait *v1alpha1.Route) (string, error) { @@ -320,135 +313,30 @@ func (r *Reconciler) createSelfsignedIssuer(ctx context.Context, routeTrait *v1a return selfSigned, fmt.Errorf("get %s err %v", selfSigned, err) } -func constructNginxIngress(routeTrait *standardv1alpha1.Route, issuer standardv1alpha1.TLS, service *runtimev1alpha1.TypedReference, port int32) *v1beta1.Ingress { - - var annotations = make(map[string]string) - - // Use nginx-ingress as implementation - annotations["kubernetes.io/ingress.class"] = "nginx" - - // SSL - var issuerAnn = "cert-manager.io/issuer" - if issuer.Type == standardv1alpha1.ClusterIssuer { - issuerAnn = "cert-manager.io/cluster-issuer" - } - annotations[issuerAnn] = issuer.IssuerName - - // Rewrite - if routeTrait.Spec.RewriteTarget != "" { - annotations["ingress.kubernetes.io/rewrite-target"] = routeTrait.Spec.RewriteTarget - } - - // Custom headers - var headerSnippet string - for k, v := range routeTrait.Spec.CustomHeaders { - headerSnippet += fmt.Sprintf("more_set_headers \"%s: %s\";\n", k, v) - } - if headerSnippet != "" { - annotations["nginx.ingress.kubernetes.io/configuration-snippet"] = headerSnippet - } - backend := routeTrait.Spec.Backend - if backend != nil { - // Backend protocol - if backend.Protocol != "" { - annotations["nginx.ingress.kubernetes.io/backend-protocol"] = backend.Protocol - } - - //Send timeout - if backend.SendTimeout != 0 { - annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = strconv.Itoa(backend.SendTimeout) - } - - //Read timeout - if backend.ReadTimeout != 0 { - annotations["nginx.ingress.kubernetes.io/proxy‑read‑timeout"] = strconv.Itoa(backend.ReadTimeout) - } - } - - ingress := &v1beta1.Ingress{ - TypeMeta: metav1.TypeMeta{ - Kind: reflect.TypeOf(v1beta1.Ingress{}).Name(), - APIVersion: v1beta1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: routeTrait.Name, - Namespace: routeTrait.Namespace, - Annotations: annotations, - Labels: oamServiceLabel, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: routeTrait.GetObjectKind().GroupVersionKind().GroupVersion().String(), - Kind: routeTrait.GetObjectKind().GroupVersionKind().Kind, - UID: routeTrait.GetUID(), - Name: routeTrait.GetName(), - Controller: pointer.BoolPtr(true), - BlockOwnerDeletion: pointer.BoolPtr(true), - }, - }, - }, - } - ingress.Spec.TLS = []v1beta1.IngressTLS{ - { - Hosts: []string{routeTrait.Spec.Host}, - SecretName: routeTrait.Name + "-cert", - }, - } - if routeTrait.Spec.DefaultBackend != nil { - ingress.Spec.Backend = routeTrait.Spec.DefaultBackend - } - ingress.Spec.Rules = []v1beta1.IngressRule{ - { - Host: routeTrait.Spec.Host, - IngressRuleValue: v1beta1.IngressRuleValue{HTTP: &v1beta1.HTTPIngressRuleValue{ - Paths: []v1beta1.HTTPIngressPath{ - { - Path: routeTrait.Spec.Path, - Backend: v1beta1.IngressBackend{ - ServiceName: service.Name, - ServicePort: intstr.FromInt(int(port)), - }, - }, - }, - }}, - }, - } - return ingress -} - // fetch the service that is associated with the workload -func (r *Reconciler) fetchService(ctx context.Context, mLog logr.Logger, - workload *unstructured.Unstructured, routeTrait *v1alpha1.Route) (*runtimev1alpha1.TypedReference, bool, int32, error) { - // Fetch the child resources list from the corresponding workload - resources, err := oamutil.FetchWorkloadChildResources(ctx, mLog, r, workload) - if err != nil { - if !apierrors.IsNotFound(err) { - mLog.Error(err, "Error while fetching the workload child resources", "workload kind", workload.GetKind(), - "workload name", workload.GetName()) - } - return nil, false, 0, err +func (r *Reconciler) fillBackendByCheckChildResource(mLog logr.Logger, + routeTrait *v1alpha1.Route, childResources []*unstructured.Unstructured) error { + if len(childResources) == 0 { + return nil } - // find the service that has the port - for _, childRes := range resources { + for _, childRes := range childResources { if childRes.GetAPIVersion() == corev1.SchemeGroupVersion.String() && childRes.GetKind() == reflect.TypeOf(corev1.Service{}).Name() { - svc := &runtimev1alpha1.TypedReference{ - APIVersion: common.ServiceAPIVersion, - Kind: common.ServiceKind, - Name: childRes.GetName(), - UID: childRes.GetUID(), + data, err := json.Marshal(childRes.Object) + if err != nil { + mLog.Error(err, "error marshal child childResources as K8s Service, continue to check other resource", "resource name", childRes.GetName()) + continue } - ports, _, _ := unstructured.NestedSlice(childRes.Object, "spec", "ports") - for _, port := range ports { - data, _ := json.Marshal(port) - var servicePort corev1.ServicePort - _ = json.Unmarshal(data, &servicePort) - if routeTrait.Spec.Backend == nil || routeTrait.Spec.Backend.Port.IntValue() == 0 || servicePort.TargetPort == routeTrait.Spec.Backend.Port { - return svc, true, servicePort.Port, nil - } + var service corev1.Service + err = json.Unmarshal(data, &service) + if err != nil { + mLog.Error(err, "error unmarshal child childResources as K8s Service, continue to check other resource", "resource name", childRes.GetName()) + continue } + FillRouteTraitWithService(&service, routeTrait) } } - return nil, false, 0, nil + return nil } func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { diff --git a/pkg/controller/v1alpha1/routes/route_controller_test.go b/pkg/controller/v1alpha1/routes/route_controller_test.go new file mode 100644 index 000000000..7a52721c6 --- /dev/null +++ b/pkg/controller/v1alpha1/routes/route_controller_test.go @@ -0,0 +1,351 @@ +package routes + +import ( + "context" + "errors" + "time" + + "github.com/oam-dev/kubevela/api/v1alpha1" + + "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2" + "github.com/crossplane/oam-kubernetes-runtime/pkg/oam/util" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + certmanager "github.com/wonderflow/cert-manager-api/pkg/apis/certmanager/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/api/networking/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var _ = Describe("Route Trait Integration Test", func() { + // common var init + ctx := context.Background() + namespaceName := "routetrait-integration-test" + + podPort := 8000 + issuerName := "my-issuer" + + var ns corev1.Namespace + getComponent := func(workloadType, compName string) (v1alpha2.Component, map[string]string, map[string]string) { + podTemplateLabel := map[string]string{"standard.oam.dev": "oam-test-deployment", "workload.oam.dev/type": workloadType} + workloadLabel := map[string]string{"standard.oam.dev": "oam-test-deployment", "app.oam.dev/component": compName, "app.oam.dev/name": "test-app-" + compName} + basedeploy := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Labels: podTemplateLabel, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: podTemplateLabel, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podTemplateLabel, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container-name", + Image: "crccheck/hello-world", + ImagePullPolicy: corev1.PullNever, + Ports: []corev1.ContainerPort{ + { + ContainerPort: int32(podPort), + }}}}}}}, + } + var rp = int32(1) + if workloadType == "webservice" { + basePodSpecc := &v1alpha1.PodSpecWorkload{ + TypeMeta: metav1.TypeMeta{ + Kind: "PodSpecWorkload", + APIVersion: "standard.oam.dev/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Labels: podTemplateLabel, + }, + Spec: v1alpha1.PodSpecWorkloadSpec{ + Replicas: &rp, + PodSpec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "container-name", + Image: "crccheck/hello-world", + ImagePullPolicy: corev1.PullNever, + Ports: []corev1.ContainerPort{ + { + ContainerPort: int32(podPort), + }}}}}}, + } + return v1alpha2.Component{ + TypeMeta: metav1.TypeMeta{ + Kind: "Component", + APIVersion: "core.oam.dev/v1alpha2", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: compName, + Namespace: ns.Name, + }, + Spec: v1alpha2.ComponentSpec{ + Workload: runtime.RawExtension{Object: basePodSpecc}, + }, + }, workloadLabel, podTemplateLabel + } + return v1alpha2.Component{ + TypeMeta: metav1.TypeMeta{ + Kind: "Component", + APIVersion: "core.oam.dev/v1alpha2", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: compName, + Namespace: ns.Name, + }, + Spec: v1alpha2.ComponentSpec{ + Workload: runtime.RawExtension{Object: basedeploy}, + }, + }, workloadLabel, podTemplateLabel + } + + getAC := func(compName string) v1alpha2.ApplicationConfiguration { + return v1alpha2.ApplicationConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "ApplicationConfiguration", + APIVersion: "core.oam.dev/v1alpha2", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: "test-app-" + compName, + }, + Spec: v1alpha2.ApplicationConfigurationSpec{ + Components: []v1alpha2.ApplicationConfigurationComponent{{ + ComponentName: compName, + Traits: []v1alpha2.ComponentTrait{ + { + Trait: runtime.RawExtension{Object: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "standard.oam.dev/v1alpha1", + "kind": "Route", + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "trait.oam.dev/type": "route", + }, + }, + "spec": map[string]interface{}{ + "host": "mycomp.mytest.com", + "tls": map[string]interface{}{ + "issuerName": issuerName, + }}}}}}}}}}} + } + BeforeEach(func() { + logf.Log.Info("[TEST] Set up resources before an integration test") + ns = corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + + By("Create the Namespace for test") + Expect(k8sClient.Create(ctx, &ns)).Should(SatisfyAny(Succeed(), &util.AlreadyExistMatcher{})) + + By("Create the Issuer for test") + Expect(k8sClient.Create(context.Background(), &certmanager.Issuer{ + ObjectMeta: metav1.ObjectMeta{Name: issuerName, Namespace: namespaceName}, + Spec: certmanager.IssuerSpec{IssuerConfig: certmanager.IssuerConfig{SelfSigned: &certmanager.SelfSignedIssuer{}}}, + })).Should(SatisfyAny(Succeed(), &util.AlreadyExistMatcher{})) + }) + + AfterEach(func() { + // Control-runtime test environment has a bug that can't delete resources like deployment/namespaces + // We have to use different names to segregate between tests + logf.Log.Info("[TEST] Clean up resources after an integration test") + }) + + It("Test with child resource no podSpecable but has service child using webservice workload", func() { + compName := "test-webservice" + comp, _, _ := getComponent("webservice", compName) + ac := getAC(compName) + Expect(k8sClient.Create(ctx, &comp)).ToNot(HaveOccurred()) + Expect(k8sClient.Create(ctx, &ac)).ToNot(HaveOccurred()) + + By("Check that we have created the route") + createdRoute := v1alpha1.Route{} + var traitName string + Eventually( + func() error { + err := k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: ac.Name}, + &ac) + if err != nil { + return err + } + if len(ac.Status.Workloads) < 1 || len(ac.Status.Workloads[0].Traits) < 1 { + return errors.New("workload or trait not ready") + } + traitName = ac.Status.Workloads[0].Traits[0].Reference.Name + err = k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: traitName}, + &createdRoute) + if err != nil { + return err + } + if len(createdRoute.Status.Ingresses) == 0 { + return errors.New("no ingress created") + } + return nil + }, + time.Second*30, time.Millisecond*500).Should(BeNil()) + + By("Check that we have created the ingress") + createdIngress := v1beta1.Ingress{} + Eventually( + func() error { + return k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: createdRoute.Status.Ingresses[0].Name}, + &createdIngress) + }, + time.Second*30, time.Millisecond*500).Should(BeNil()) + logf.Log.Info("[TEST] Get the created ingress", "ingress rules", createdIngress.Spec.Rules) + Expect(createdIngress.GetNamespace()).Should(Equal(namespaceName)) + Expect(len(createdIngress.Spec.Rules)).Should(Equal(1)) + Expect(createdIngress.Spec.Rules[0].Host).Should(Equal("mycomp.mytest.com")) + Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName).Should(Equal(compName)) + Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort.IntVal).Should(Equal(int32(8080))) + }) + It("Test with podSpec label with no podSpecPath using deployment workload", func() { + compName := "test-deployment" + comp, _, deploylabel := getComponent("deployment", compName) + ac := getAC(compName) + Expect(k8sClient.Create(ctx, &comp)).ToNot(HaveOccurred()) + Expect(k8sClient.Create(ctx, &ac)).ToNot(HaveOccurred()) + + By("Check that we have created the route") + createdRoute := v1alpha1.Route{} + var traitName string + Eventually( + func() error { + err := k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: ac.Name}, + &ac) + if err != nil { + return err + } + if len(ac.Status.Workloads) < 1 || len(ac.Status.Workloads[0].Traits) < 1 { + return errors.New("workload or trait not ready") + } + traitName = ac.Status.Workloads[0].Traits[0].Reference.Name + err = k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: traitName}, + &createdRoute) + if err != nil { + return err + } + if len(createdRoute.Status.Ingresses) == 0 { + return errors.New("no ingress created") + } + return nil + }, + time.Second*30, time.Millisecond*500).Should(BeNil()) + + By("Check that we have created the ingress") + createdIngress := v1beta1.Ingress{} + Eventually( + func() error { + return k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: createdRoute.Status.Ingresses[0].Name}, + &createdIngress) + }, + time.Second*30, time.Millisecond*500).Should(BeNil()) + logf.Log.Info("[TEST] Get the created ingress", "ingress rules", createdIngress.Spec.Rules) + Expect(createdIngress.GetNamespace()).Should(Equal(namespaceName)) + Expect(len(createdIngress.Spec.Rules)).Should(Equal(1)) + Expect(createdIngress.Spec.Rules[0].Host).Should(Equal("mycomp.mytest.com")) + Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName).Should(Equal(traitName)) + Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort.IntVal).Should(Equal(int32(8000))) + + By("Check that we have created the service") + createdSvc := corev1.Service{} + Eventually( + func() error { + return k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: traitName}, + &createdSvc) + }, + time.Second*30, time.Millisecond*500).Should(BeNil()) + logf.Log.Info("[TEST] Get the created service", "service ports", createdSvc.Spec.Ports) + Expect(createdSvc.Spec.Selector).Should(Equal(deploylabel)) + Expect(createdSvc.Spec.Ports[0].TargetPort.IntVal).Should(Equal(int32(podPort))) + }) + It("Test with podSpecPath specified using deploy workload", func() { + compName := "test-deploy" + comp, workloadLabel, _ := getComponent("deploy", compName) + ac := getAC(compName) + Expect(k8sClient.Create(ctx, &comp)).ToNot(HaveOccurred()) + Expect(k8sClient.Create(ctx, &ac)).ToNot(HaveOccurred()) + + By("Check that we have created the route") + createdRoute := v1alpha1.Route{} + var traitName string + Eventually( + func() error { + err := k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: ac.Name}, + &ac) + if err != nil { + return err + } + if len(ac.Status.Workloads) < 1 || len(ac.Status.Workloads[0].Traits) < 1 { + return errors.New("workload or trait not ready") + } + traitName = ac.Status.Workloads[0].Traits[0].Reference.Name + err = k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: traitName}, + &createdRoute) + if err != nil { + return err + } + if len(createdRoute.Status.Ingresses) == 0 { + return errors.New("no ingress created") + } + return nil + }, + time.Second*30, time.Millisecond*500).Should(BeNil()) + + By("Check that we have created the ingress") + createdIngress := v1beta1.Ingress{} + Eventually( + func() error { + return k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: createdRoute.Status.Ingresses[0].Name}, + &createdIngress) + }, + time.Second*30, time.Millisecond*500).Should(BeNil()) + logf.Log.Info("[TEST] Get the created ingress", "ingress rules", createdIngress.Spec.Rules) + Expect(createdIngress.GetNamespace()).Should(Equal(namespaceName)) + Expect(len(createdIngress.Spec.Rules)).Should(Equal(1)) + Expect(createdIngress.Spec.Rules[0].Host).Should(Equal("mycomp.mytest.com")) + Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName).Should(Equal(traitName)) + Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort.IntVal).Should(Equal(int32(8000))) + + By("Check that we have created the service") + createdSvc := corev1.Service{} + Eventually( + func() error { + return k8sClient.Get(ctx, + types.NamespacedName{Namespace: ns.Name, Name: traitName}, + &createdSvc) + }, + time.Second*30, time.Millisecond*500).Should(BeNil()) + logf.Log.Info("[TEST] Get the created service", "service ports", createdSvc.Spec.Ports) + for k, v := range workloadLabel { + Expect(createdSvc.Spec.Selector).Should(HaveKeyWithValue(k, v)) + } + Expect(createdSvc.Spec.Ports[0].TargetPort.IntVal).Should(Equal(int32(podPort))) + }) +}) diff --git a/pkg/controller/v1alpha1/routes/suite_test.go b/pkg/controller/v1alpha1/routes/suite_test.go index 0946de266..70c2e4bdb 100644 --- a/pkg/controller/v1alpha1/routes/suite_test.go +++ b/pkg/controller/v1alpha1/routes/suite_test.go @@ -17,13 +17,27 @@ limitations under the License. package routes import ( + "context" "path/filepath" "testing" + "github.com/oam-dev/kubevela/pkg/controller/v1alpha1/podspecworkload" + + "github.com/crossplane/crossplane-runtime/pkg/logging" + + "github.com/crossplane/oam-kubernetes-runtime/pkg/controller" + "github.com/crossplane/oam-kubernetes-runtime/pkg/controller/v1alpha2/applicationconfiguration" + + oamCore "github.com/crossplane/oam-kubernetes-runtime/apis/core" + "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + certmanager "github.com/wonderflow/cert-manager-api/pkg/apis/certmanager/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" @@ -40,6 +54,10 @@ import ( var cfg *rest.Config var k8sClient client.Client var testEnv *envtest.Environment +var controllerDone chan struct{} +var routeNS corev1.Namespace + +var RouteNSName = "route-test" func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) @@ -50,34 +68,103 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func(done Done) { - logf.SetLogger(zap.LoggerTo(GinkgoWriter, true)) - - By("bootstrapping test environment") - useExistCluster := true + logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter))) + routeNS = corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: RouteNSName, + }, + } + By("Bootstrapping test environment") + useExistCluster := false testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{ + filepath.Join("../../../..", "charts/vela-core/crds"), // this has all the required CRDs, + }, UseExistingCluster: &useExistCluster, } - var err error cfg, err = testEnv.Start() Expect(err).ToNot(HaveOccurred()) Expect(cfg).ToNot(BeNil()) - err = standardv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) + Expect(standardv1alpha1.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred()) + Expect(oamCore.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred()) + Expect(certmanager.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme - + By("Create the k8s client") k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).ToNot(HaveOccurred()) Expect(k8sClient).ToNot(BeNil()) + By("Starting the route trait controller in the background") + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Port: 9443, + }) + Expect(err).ToNot(HaveOccurred()) + r := Reconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("RouteTrait"), + Scheme: mgr.GetScheme(), + } + Expect(r.SetupWithManager(mgr)).ToNot(HaveOccurred()) + Expect(applicationconfiguration.Setup(mgr, controller.Args{}, logging.NewLogrLogger(ctrl.Log.WithName("AppConfig")))).ToNot(HaveOccurred()) + Expect(podspecworkload.Setup(mgr)).ToNot(HaveOccurred()) + + controllerDone = make(chan struct{}, 1) + // +kubebuilder:scaffold:builder + go func() { + defer GinkgoRecover() + Expect(mgr.Start(controllerDone)).ToNot(HaveOccurred()) + }() + + By("Create the routeTrait namespace") + Expect(k8sClient.Create(context.Background(), &routeNS)).ToNot(HaveOccurred()) + routeDef := &v1alpha2.TraitDefinition{} + routeDef.Name = "route" + routeDef.Namespace = RouteNSName + routeDef.Spec.Reference.Name = "routes.standard.oam.dev" + routeDef.Spec.WorkloadRefPath = "spec.workloadRef" + Expect(k8sClient.Create(context.Background(), routeDef)).ToNot(HaveOccurred()) + + webservice := &v1alpha2.WorkloadDefinition{} + webservice.Name = "webservice" + webservice.Namespace = RouteNSName + webservice.Spec.Reference.Name = "podspecworkloads.standard.oam.dev" + webservice.Spec.ChildResourceKinds = []v1alpha2.ChildResourceKind{{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, { + APIVersion: "v1", + Kind: "Service", + }} + Expect(k8sClient.Create(context.Background(), webservice)).ToNot(HaveOccurred()) + + deployment := &v1alpha2.WorkloadDefinition{} + deployment.Name = "deployment" + deployment.Namespace = RouteNSName + deployment.Labels = map[string]string{"workload.oam.dev/podspecable": "true"} + deployment.Spec.Reference.Name = "deployments.apps" + Expect(k8sClient.Create(context.Background(), deployment)).ToNot(HaveOccurred()) + + deploy := &v1alpha2.WorkloadDefinition{} + deploy.Name = "deploy" + deploy.Namespace = RouteNSName + deploy.Spec.PodSpecPath = "spec.template.spec" + deploy.Spec.Reference.Name = "deployments.apps" + Expect(k8sClient.Create(context.Background(), deploy)).ToNot(HaveOccurred()) + close(done) }, 60) var _ = AfterSuite(func() { - By("tearing down the test environment") + By("Stop the routeTrait controller") + close(controllerDone) + By("Delete the route-test namespace") + Expect(k8sClient.Delete(context.Background(), &routeNS, + client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed()) + By("Tearing down the test environment") err := testEnv.Stop() Expect(err).ToNot(HaveOccurred()) }) diff --git a/pkg/controller/v1alpha1/routes/util.go b/pkg/controller/v1alpha1/routes/util.go new file mode 100644 index 000000000..71ea5ba71 --- /dev/null +++ b/pkg/controller/v1alpha1/routes/util.go @@ -0,0 +1,161 @@ +package routes + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/crossplane/oam-kubernetes-runtime/pkg/oam" + + "github.com/oam-dev/kubevela/api/types" + "github.com/oam-dev/kubevela/api/v1alpha1" + + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" + "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func NeedDiscovery(routeTrait *v1alpha1.Route) bool { + if len(routeTrait.Spec.Rules) == 0 { + return true + } + for _, rule := range routeTrait.Spec.Rules { + if rule.Backend == nil { + return true + } + if rule.Backend.BackendService == nil { + return true + } + if rule.Backend.BackendService.ServiceName == "" { + return true + } + } + return false +} + +func GetPodSpecPath(workloadDef *v1alpha2.WorkloadDefinition) (string, bool) { + if workloadDef.Spec.PodSpecPath != "" { + return workloadDef.Spec.PodSpecPath, true + } + if workloadDef.Labels == nil { + return "", false + } + podSpecable, ok := workloadDef.Labels[types.LabelPodSpecable] + if !ok { + return "", false + } + ok, _ = strconv.ParseBool(podSpecable) + return "", ok +} + +func discoveryFromPodSpec(w *unstructured.Unstructured, fieldPath string) ([]intstr.IntOrString, error) { + paved := fieldpath.Pave(w.Object) + obj, err := paved.GetValue(fieldPath) + if err != nil { + return nil, err + } + data, err := json.Marshal(obj) + if err != nil { + return nil, fmt.Errorf("discovery podSpec from %s in workload %v err %v", fieldPath, w.GetName(), err) + } + var spec corev1.PodSpec + err = json.Unmarshal(data, &spec) + if err != nil { + return nil, fmt.Errorf("discovery podSpec from %s in workload %v err %v", fieldPath, w.GetName(), err) + } + ports := getContainerPorts(spec.Containers) + if len(ports) == 0 { + return nil, fmt.Errorf("no port found in podSpec %v", w.GetName()) + } + return ports, nil +} + +// discoveryFromPodTemplate not only discovery port, will also use labels in podTemplate +func discoveryFromPodTemplate(w *unstructured.Unstructured, fields ...string) ([]intstr.IntOrString, map[string]string, error) { + obj, found, _ := unstructured.NestedMap(w.Object, fields...) + if !found { + return nil, nil, fmt.Errorf("not have spec.template in workload %v", w.GetName()) + } + data, err := json.Marshal(obj) + if err != nil { + return nil, nil, fmt.Errorf("workload %v convert object err %v", w.GetName(), err) + } + var spec corev1.PodTemplateSpec + err = json.Unmarshal(data, &spec) + if err != nil { + return nil, nil, fmt.Errorf("workload %v convert object to PodTemplate err %v", w.GetName(), err) + } + ports := getContainerPorts(spec.Spec.Containers) + if len(ports) == 0 { + return nil, nil, fmt.Errorf("no port found in workload %v", w.GetName()) + } + return ports, spec.Labels, nil +} + +func getContainerPorts(cs []corev1.Container) []intstr.IntOrString { + var ports []intstr.IntOrString + //TODO(wonderflow): exclude some sidecars + for _, container := range cs { + for _, port := range container.Ports { + ports = append(ports, intstr.FromInt(int(port.ContainerPort))) + } + } + return ports +} + +// MatchService try check if the service matches the rules +func MatchService(targetPort intstr.IntOrString, rule v1alpha1.Rule) bool { + // the rule is nil, continue + if rule.Backend == nil || rule.Backend.BackendService == nil || rule.Backend.BackendService.Port.IntValue() == 0 { + return true + } + if rule.Backend.BackendService.ServiceName != "" { + return false + } + // the rule is not null, if any port matches, we regard them are all match + if targetPort == rule.Backend.BackendService.Port { + return true + } + // port is not matched, mark it not match + return false +} + +func FillRouteTraitWithService(service *corev1.Service, routeTrait *v1alpha1.Route) { + if len(routeTrait.Spec.Rules) == 0 { + routeTrait.Spec.Rules = []v1alpha1.Rule{{Name: "auto-created"}} + } + for idx, rule := range routeTrait.Spec.Rules { + // If backendService.port not specified, will always use the service found and it's first port as backendService. + for _, servicePort := range service.Spec.Ports { + //We use targetPort rather than port to match with the rule, because if serviceName not specified, + //Users will only know containerPort(which is targetPort) + if MatchService(servicePort.TargetPort, rule) { + ref := &v1alpha1.BackendServiceRef{ + //Use port of service rather than targetPort, it will be used in ingress pointing to the service + Port: intstr.FromInt(int(servicePort.Port)), + ServiceName: service.Name, + } + if rule.Backend == nil { + rule.Backend = &v1alpha1.Backend{BackendService: ref} + } else { + rule.Backend.BackendService = ref + } + routeTrait.Spec.Rules[idx] = rule + break + } + } + } +} + +func filterLabels(labels map[string]string) map[string]string { + newLabel := make(map[string]string) + for k, v := range labels { + if k == oam.LabelOAMResourceType || k == oam.WorkloadTypeLabel { + continue + } + newLabel[k] = v + } + return newLabel +} diff --git a/pkg/controller/v1alpha1/routes/util_test.go b/pkg/controller/v1alpha1/routes/util_test.go new file mode 100644 index 000000000..0db51ae52 --- /dev/null +++ b/pkg/controller/v1alpha1/routes/util_test.go @@ -0,0 +1 @@ +package routes diff --git a/pkg/oam/trait.go b/pkg/oam/trait.go index 9f79ef09e..c495c7260 100644 --- a/pkg/oam/trait.go +++ b/pkg/oam/trait.go @@ -2,7 +2,6 @@ package oam import ( "context" - "encoding/json" "fmt" "strconv" "strings" @@ -11,8 +10,6 @@ import ( plur "github.com/gertd/go-pluralize" "github.com/gin-gonic/gin" "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/api/types" @@ -21,21 +18,6 @@ import ( "github.com/oam-dev/kubevela/pkg/server/apis" ) -func GetTraitDefNameFromRaw(extension runtime.RawExtension) string { - if extension.Raw == nil { - extension.Raw, _ = extension.MarshalJSON() - } - var data map[string]interface{} - // leverage Admission Controller to do the check - _ = json.Unmarshal(extension.Raw, &data) - obj := unstructured.Unstructured{Object: data} - ann := obj.GetAnnotations() - if ann == nil { - return obj.GetKind() - } - return ann[types.AnnTraitDef] -} - func ListTraitDefinitions(workloadName *string) ([]types.Capability, error) { var traitList []types.Capability traits, err := plugins.LoadInstalledCapabilityWithType(types.TypeTrait)