github/kubevela
1
0
Fork 0
mirror of https://github.com/kubevela/kubevela.git synced 2026-08-27 16:17:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #387 from wonderflow/route1

Browse Source 
give route trait new discovery port way with podspecable design
This commit is contained in:
Jianbo Sun
2020-10-18 11:24:38 +08:00
committed by GitHub
parent b08c6b9441 3d2bd9a898
commit 1cc8a3980c
30 changed files with 1591 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1
View File
@@ -128,6 +128,7 @@ core-uninstall: manifests
# Generate manifests e.g. CRD, RBAC etc.
manifests: controller-gen
$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=charts/vela-core/crds
rm charts/vela-core/crds/_.yaml
# Generate code
generate: controller-gen
README.md
+2
View File
@@ -150,6 +150,8 @@ Let's take `route` as example.
### `route`
If you want to use `route`, please make sure you have [nginx-ingress controller[https://kubernetes.github.io/ingress-nginx/deploy/] in your cluster.
```console
$ vela route mycomp --app myapp
Adding route for app mycomp
api/types/types.go
+1 -4
View File
@@ -15,10 +15,7 @@ const (
AnnKind = "definition.oam.dev/kind"
AnnDescription = "definition.oam.dev/description"
// Indicate which workloadDefinition generate from
AnnWorkloadDef = "workload.oam.dev/name"
// Indicate which traitDefinition generate from
AnnTraitDef = "trait.oam.dev/name"
LabelPodSpecable = "workload.oam.dev/podspecable"
)
const (
api/v1alpha1/route_types.go
+34 -19
View File
@@ -19,7 +19,6 @@ package v1alpha1
import (
runtimev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1"
"github.com/crossplane/oam-kubernetes-runtime/pkg/oam"
"k8s.io/api/networking/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
@@ -34,15 +33,24 @@ type RouteSpec struct {
// Host is the host of the route
Host string `json:"host"`
// Path is location Path, default for "/"
Path string `json:"path,omitempty"`
// TLS indicate route trait will create SSL secret using cert-manager with specified issuer
// If this is nil, route trait will use a selfsigned issuer
TLS *TLS `json:"tls,omitempty"`
// DefaultBackend uses serviceName
DefaultBackend *v1beta1.IngressBackend `json:"defaultBackend,omitempty"`
// Rules contain multiple rules of route
Rules []Rule `json:"rules,omitempty"`
// Provider indicate which ingress controller implementation the route trait will use, by default it's nginx-ingress
Provider string `json:"provider,omitempty"`
}
// Rule defines to route rule
type Rule struct {
// Name will become the suffix of underlying ingress created by this rule, if not, will use index as suffix.
Name string `json:"name,omitempty"`
// Path is location Path, default for "/"
Path string `json:"path,omitempty"`
// RewriteTarget will rewrite request from Path to RewriteTarget path.
RewriteTarget string `json:"rewriteTarget,omitempty"`
@@ -50,6 +58,9 @@ type RouteSpec struct {
// CustomHeaders pass a custom list of headers to the backend service.
CustomHeaders map[string]string `json:"customHeaders,omitempty"`
// DefaultBackend will become the ingress default backend if the backend is not available
DefaultBackend *runtimev1alpha1.TypedReference `json:"defaultBackend,omitempty"`
// Backend indicate how to connect backend service
// If it's nil, will auto discovery
Backend *Backend `json:"backend,omitempty"`
@@ -58,7 +69,8 @@ type RouteSpec struct {
type TLS struct {
IssuerName string `json:"issuerName,omitempty"`
// Type indicate the issuer is ClusterIssuer or NamespaceIssuer
// Type indicate the issuer is ClusterIssuer or Issuer(namespace issuer), by default, it's Issuer
// +kubebuilder:default:=Issuer
Type IssuerType `json:"type,omitempty"`
}
@@ -69,27 +81,30 @@ const (
NamespaceIssuer IssuerType = "Issuer"
)
// Route will automatically discover podTemplate for Port and SelectLabels if they are not set.
// If Port and SelectLabels are already set, discovery won't work.
// If Port is not set, the first port discovered will be set.
// If SelectLabels are not set, all selectorLabels discovered will be set.
// Route will automatically discover podSpec and label for BackendService.
// If BackendService is already set, discovery won't work.
// If BackendService is not set, the discovery mechanism will work.
type Backend struct {
// Protocol means backend-protocol, HTTP, HTTPS, GRPC, GRPCS, AJP and FCGI, By default uses HTTP
Protocol string `json:"protocol,omitempty"`
// ReadTimeout used for setting read timeout duration for backend service, the unit is second.
ReadTimeout int `json:"readTimeout,omitempty"`
// SendTimeout used for setting send timeout duration for backend service, the unit is second.
SendTimeout int `json:"sendTimeout,omitempty"`
// Port points to backend service port.
Port intstr.IntOrString `json:"port,omitempty"`
// SelectLabels for backend service.
SelectLabels map[string]string `json:"selectLabels,omitempty"`
// BackendService specifies the backend K8s service and port, it's optional
BackendService *BackendServiceRef `json:"backendService,omitempty"`
}
// BackendServiceRef specifies the backend K8s service and port, if specified, the two fields are all required
type BackendServiceRef struct {
// Port allow you direct specify backend service port.
Port intstr.IntOrString `json:"port"`
// ServiceName allow you direct specify K8s service for backend service.
ServiceName string `json:"serviceName"`
}
// RouteStatus defines the observed state of Route
type RouteStatus struct {
Ingress *runtimev1alpha1.TypedReference `json:"ingress,omitempty"`
Service *runtimev1alpha1.TypedReference `json:"service,omitempty"`
Ingresses []runtimev1alpha1.TypedReference `json:"ingresses,omitempty"`
Service *runtimev1alpha1.TypedReference `json:"service,omitempty"`
runtimev1alpha1.ConditionedStatus `json:",inline"`
}
api/v1alpha1/zz_generated.deepcopy.go
+61 -27
View File
@@ -22,20 +22,16 @@ package v1alpha1
import (
corev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1"
"k8s.io/api/networking/v1beta1"
runtime "k8s.io/apimachinery/pkg/runtime"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Backend) DeepCopyInto(out *Backend) {
*out = *in
out.Port = in.Port
if in.SelectLabels != nil {
in, out := &in.SelectLabels, &out.SelectLabels
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
if in.BackendService != nil {
in, out := &in.BackendService, &out.BackendService
*out = new(BackendServiceRef)
**out = **in
}
}
@@ -49,6 +45,22 @@ func (in *Backend) DeepCopy() *Backend {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BackendServiceRef) DeepCopyInto(out *BackendServiceRef) {
*out = *in
out.Port = in.Port
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendServiceRef.
func (in *BackendServiceRef) DeepCopy() *BackendServiceRef {
if in == nil {
return nil
}
out := new(BackendServiceRef)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *MetricsTrait) DeepCopyInto(out *MetricsTrait) {
*out = *in
@@ -315,23 +327,13 @@ func (in *RouteSpec) DeepCopyInto(out *RouteSpec) {
*out = new(TLS)
**out = **in
}
if in.DefaultBackend != nil {
in, out := &in.DefaultBackend, &out.DefaultBackend
*out = new(v1beta1.IngressBackend)
(*in).DeepCopyInto(*out)
}
if in.CustomHeaders != nil {
in, out := &in.CustomHeaders, &out.CustomHeaders
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
if in.Rules != nil {
in, out := &in.Rules, &out.Rules
*out = make([]Rule, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.Backend != nil {
in, out := &in.Backend, &out.Backend
*out = new(Backend)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteSpec.
@@ -347,10 +349,10 @@ func (in *RouteSpec) DeepCopy() *RouteSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *RouteStatus) DeepCopyInto(out *RouteStatus) {
*out = *in
if in.Ingress != nil {
in, out := &in.Ingress, &out.Ingress
*out = new(corev1alpha1.TypedReference)
**out = **in
if in.Ingresses != nil {
in, out := &in.Ingresses, &out.Ingresses
*out = make([]corev1alpha1.TypedReference, len(*in))
copy(*out, *in)
}
if in.Service != nil {
in, out := &in.Service, &out.Service
@@ -370,6 +372,38 @@ func (in *RouteStatus) DeepCopy() *RouteStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Rule) DeepCopyInto(out *Rule) {
*out = *in
if in.CustomHeaders != nil {
in, out := &in.CustomHeaders, &out.CustomHeaders
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.DefaultBackend != nil {
in, out := &in.DefaultBackend, &out.DefaultBackend
*out = new(corev1alpha1.TypedReference)
**out = **in
}
if in.Backend != nil {
in, out := &in.Backend, &out.Backend
*out = new(Backend)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Rule.
func (in *Rule) DeepCopy() *Rule {
if in == nil {
return nil
}
out := new(Rule)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ScapeServiceEndPoint) DeepCopyInto(out *ScapeServiceEndPoint) {
*out = *in
charts/vela-core/crds/core.oam.dev_applicationconfigurations.yaml
-2
View File
@@ -547,8 +547,6 @@ spec:
type: object
type: object
type: array
required:
- historyWorkloads
type: object
type: object
served: true
charts/vela-core/crds/standard.oam.dev_routes.yaml
+107 -97
View File
@@ -38,84 +38,90 @@ spec:
spec:
description: RouteSpec defines the desired state of Route
properties:
backend:
description: Backend indicate how to connect backend service If it's
nil, will auto discovery
properties:
port:
anyOf:
- type: integer
- type: string
description: Port points to backend service port.
x-kubernetes-int-or-string: true
protocol:
description: Protocol means backend-protocol, HTTP, HTTPS, GRPC,
GRPCS, AJP and FCGI, By default uses HTTP
type: string
readTimeout:
description: ReadTimeout used for setting read timeout duration
for backend service, the unit is second.
type: integer
selectLabels:
additionalProperties:
type: string
description: SelectLabels for backend service.
type: object
sendTimeout:
description: SendTimeout used for setting send timeout duration
for backend service, the unit is second.
type: integer
type: object
customHeaders:
additionalProperties:
type: string
description: CustomHeaders pass a custom list of headers to the backend
service.
type: object
defaultBackend:
description: DefaultBackend uses serviceName
properties:
resource:
description: Resource is an ObjectRef to another Kubernetes resource
in the namespace of the Ingress object. If resource is specified,
serviceName and servicePort must not be specified.
properties:
apiGroup:
description: APIGroup is the group for the resource being
referenced. If APIGroup is not specified, the specified
Kind must be in the core API group. For any other third-party
types, APIGroup is required.
type: string
kind:
description: Kind is the type of resource being referenced
type: string
name:
description: Name is the name of resource being referenced
type: string
required:
- kind
- name
type: object
serviceName:
description: Specifies the name of the referenced service.
type: string
servicePort:
anyOf:
- type: integer
- type: string
description: Specifies the port of the referenced service.
x-kubernetes-int-or-string: true
type: object
host:
description: Host is the host of the route
type: string
path:
description: Path is location Path, default for "/"
type: string
rewriteTarget:
description: RewriteTarget will rewrite request from Path to RewriteTarget
path.
provider:
description: Provider indicate which ingress controller implementation
the route trait will use, by default it's nginx-ingress
type: string
rules:
description: Rules contain multiple rules of route
items:
description: Rule defines to route rule
properties:
backend:
description: Backend indicate how to connect backend service
If it's nil, will auto discovery
properties:
backendService:
description: BackendService specifies the backend K8s service
and port, it's optional
properties:
port:
anyOf:
- type: integer
- type: string
description: Port allow you direct specify backend service
port.
x-kubernetes-int-or-string: true
serviceName:
description: ServiceName allow you direct specify K8s
service for backend service.
type: string
required:
- port
- serviceName
type: object
readTimeout:
description: ReadTimeout used for setting read timeout duration
for backend service, the unit is second.
type: integer
sendTimeout:
description: SendTimeout used for setting send timeout duration
for backend service, the unit is second.
type: integer
type: object
customHeaders:
additionalProperties:
type: string
description: CustomHeaders pass a custom list of headers to
the backend service.
type: object
defaultBackend:
description: DefaultBackend will become the ingress default
backend if the backend is not available
properties:
apiVersion:
description: APIVersion of the referenced object.
type: string
kind:
description: Kind of the referenced object.
type: string
name:
description: Name of the referenced object.
type: string
uid:
description: UID of the referenced object.
type: string
required:
- apiVersion
- kind
- name
type: object
name:
description: Name will become the suffix of underlying ingress
created by this rule, if not, will use index as suffix.
type: string
path:
description: Path is location Path, default for "/"
type: string
rewriteTarget:
description: RewriteTarget will rewrite request from Path to
RewriteTarget path.
type: string
type: object
type: array
tls:
description: TLS indicate route trait will create SSL secret using
cert-manager with specified issuer If this is nil, route trait will
@@ -124,7 +130,9 @@ spec:
issuerName:
type: string
type:
description: Type indicate the issuer is ClusterIssuer or NamespaceIssuer
default: Issuer
description: Type indicate the issuer is ClusterIssuer or Issuer(namespace
issuer), by default, it's Issuer
type: string
type: object
workloadRef:
@@ -187,28 +195,30 @@ spec:
- type
type: object
type: array
ingress:
description: A TypedReference refers to an object by Name, Kind, and
APIVersion. It is commonly used to reference cluster-scoped objects
or objects where the namespace is already known.
properties:
apiVersion:
description: APIVersion of the referenced object.
type: string
kind:
description: Kind of the referenced object.
type: string
name:
description: Name of the referenced object.
type: string
uid:
description: UID of the referenced object.
type: string
required:
- apiVersion
- kind
- name
type: object
ingresses:
items:
description: A TypedReference refers to an object by Name, Kind,
and APIVersion. It is commonly used to reference cluster-scoped
objects or objects where the namespace is already known.
properties:
apiVersion:
description: APIVersion of the referenced object.
type: string
kind:
description: Kind of the referenced object.
type: string
name:
description: Name of the referenced object.
type: string
uid:
description: UID of the referenced object.
type: string
required:
- apiVersion
- kind
- name
type: object
type: array
service:
description: A TypedReference refers to an object by Name, Kind, and
APIVersion. It is commonly used to reference cluster-scoped objects
charts/vela-core/templates/definitions/manualscalars.yaml
+1 -1
View File
@@ -5,7 +5,7 @@ metadata:
definition.oam.dev/apiVersion: core.oam.dev/v1alpha2
definition.oam.dev/kind: ManualScalerTrait
definition.oam.dev/description: "Scale replica for workload"
name: manualscalertraits.core.oam.dev
name: scaler
namespace: default
spec:
appliesToWorkloads:
charts/vela-core/templates/definitions/metricstraits.yaml
+1 -1
View File
@@ -10,7 +10,7 @@ metadata:
apiVersion: core.oam.dev/v1alpha2
kind: TraitDefinition
metadata:
name: metricstraits.standard.oam.dev
name: metric
namespace: default
annotations:
definition.oam.dev/apiVersion: standard.oam.dev/v1alpha1
charts/vela-core/templates/definitions/routetrait.yaml
+1 -1
View File
@@ -1,7 +1,7 @@
apiVersion: core.oam.dev/v1alpha2
kind: TraitDefinition
metadata:
name: routes.standard.oam.dev
name: route
annotations:
definition.oam.dev/apiVersion: standard.oam.dev/v1alpha1
definition.oam.dev/kind: Route
config/rbac/role.yaml
-20
View File
@@ -74,23 +74,3 @@ rules:
- get
- patch
- update
- apiGroups:
- standard.oam.dev
resources:
- routes
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- standard.oam.dev
resources:
- routes/status
verbs:
- get
- patch
- update
design/route.md
+126
View File
@@ -0,0 +1,126 @@
# Route Trait Design
The main idea of route trait is to let users have an entrypoint to visit their App.
In k8s world, if you want to do so, you have to understand K8s [Serivce](https://kubernetes.io/docs/concepts/services-networking/service/)
, [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/), [Ingress Controllers](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/).
It's not easy to get all of these things work well.
The route trait will help you setup service, ingress automatically according your workload, along with the mTLS enabled.
The schema is also clean and easy to understand.
```go
// RouteSpec defines the desired state of Route
type RouteSpec struct {
// WorkloadReference to the workload whose metrics needs to be exposed
WorkloadReference runtimev1alpha1.TypedReference `json:"workloadRef,omitempty"`
// Host is the host of the route
Host string `json:"host"`
// TLS indicate route trait will create SSL secret using cert-manager with specified issuer
// If this is nil, route trait will use a selfsigned issuer
TLS *TLS `json:"tls,omitempty"`
// Rules contain multiple rules of route
Rules []Rule `json:"rules"`
// Provider indicate which ingress controller implementation the route trait will use, by default it's nginx-ingress
Provider string `json:"provider,omitempty"`
}
// Rule defines to route rule
type Rule struct {
// Path is location Path, default for "/"
Path string `json:"path,omitempty"`
// RewriteTarget will rewrite request from Path to RewriteTarget path.
RewriteTarget string `json:"rewriteTarget,omitempty"`
// CustomHeaders pass a custom list of headers to the backend service.
CustomHeaders map[string]string `json:"customHeaders,omitempty"`
// DefaultBackend will become the ingress default backend if the backend is not available
DefaultBackend runtimev1alpha1.TypedReference `json:"defaultBackend,omitempty"`
// Backend indicate how to connect backend service
// If it's nil, will auto discovery
Backend *Backend `json:"backend,omitempty"`
}
type TLS struct {
IssuerName string `json:"issuerName,omitempty"`
// Type indicate the issuer is ClusterIssuer or NamespaceIssuer
Type IssuerType `json:"type,omitempty"`
}
type IssuerType string
const (
ClusterIssuer IssuerType = "ClusterIssuer"
NamespaceIssuer IssuerType = "Issuer"
)
// Route will automatically discover podSpec and label for BackendService.
// If BackendService is already set, discovery won't work.
// If BackendService is not set, the discovery mechanism will work.
type Backend struct {
// ReadTimeout used for setting read timeout duration for backend service, the unit is second.
ReadTimeout int `json:"readTimeout,omitempty"`
// SendTimeout used for setting send timeout duration for backend service, the unit is second.
SendTimeout int `json:"sendTimeout,omitempty"`
// BackendService specifies the backend K8s service and port
BackendService *BackendServiceRef `json:"backendService,omitempty"`
}
type BackendServiceRef struct {
// Port allow you direct specify backend service port.
Port intstr.IntOrString `json:"port,omitempty"`
// ServiceName allow you direct specify K8s service for backend service.
ServiceName string `json:"serviceName,omitempty"`
}
```
Route Trait specifies a target workload by using `workloadRef`, in OAM system, this field will be filled automatically
by OAM runtime.
Besides `workloadRef`, one Route will have only one `host` and many rules. `host` is actually your app's visiting URL.
It's required and will be used to generate mTLS secrets.
Route Trait designed to be compatible with different ingress controller implementations, the `provider` field will allow
you to give a specified ingress controller type. Currently, only nginx-ingress is supported.
The `tls` field allow you to specify a TLS for this route with an IssuerName, the IssuerName pointing to an Issuer Object
created by cert-manager. Cert-manager and ingress controller will handle certificate creation and binding.
If not specified, a selfsigned issuer will be created.
If no rule specified, route trait will create one rule automatically and match with the port.
For every rule, we will create an ingress. In the rule, you could specify `path`, `rewriteTarget`, `customHeaders`
and `defaultBackend`. All rules will using the same `tls`, `host` and `provider`.
`defaultBackend` will become the ingress default backend with K8s Object(apiVersion/kind/name).
`backend` of the rule is completely optional.
If backendService is specified, it will use it as backend of this rule. If not,
the route trait can automatically discovery backend settings from workload.
## Discovery mechanism
1. Check ChildResource of the workload, if there already has an existing K8s service match the Backend port, use it.
2. If there's no k8s service, this means we need to create one. In order to create K8s service, we need two information
`Container Port` and `Pod SelectorLabels`.
- 2.1 Use [`PodSpecable` mechanism](https://github.com/crossplane/oam-kubernetes-runtime/blob/master/design/one-pager-podspecable-workload.md),
route trait will check `WorkloadDefinition` for podSpec field, with the `podSpec` field, we can easily find the container port.
* If podSpecPath` is specified, we will use the workload labels as `Pod SelectorLabels`.
* If `workload.oam.dev/podspecable: true` but no `podSpecPath`, will use `spec.Template` as `PodTemplate`, which means
we can get `Pod SelectorLabels` from `spec.Template.Labels`.
- 2.2 Use ChildResource: If No `PodSpecable` mechanism found in workload, we will continue discovery child resources of workload. If there
is a valid `PodTemplate` structure in child resource, we will regard it as discovery target, use the same strategy like
`workload.oam.dev/podspecable: true` but no `podSpecPath`.
documentation/samples/route-demo/definition.yaml
+53
View File
@@ -0,0 +1,53 @@
apiVersion: core.oam.dev/v1alpha2
kind: TraitDefinition
metadata:
name: route
annotations:
definition.oam.dev/apiVersion: standard.oam.dev/v1alpha1
definition.oam.dev/kind: Route
definition.oam.dev/description: "Add a route for workload"
spec:
appliesToWorkloads:
- core.oam.dev/v1alpha2.ContainerizedWorkload
- standard.oam.dev/v1alpha1.PodSpecWorkload
- deployments.apps
- webservice
workloadRefPath: spec.workloadRef
definitionRef:
name: routes.standard.oam.dev
---
apiVersion: core.oam.dev/v1alpha2
kind: WorkloadDefinition
metadata:
name: webservice
annotations:
definition.oam.dev/apiVersion: "standard.oam.dev/v1alpha1"
definition.oam.dev/kind: "PodSpecWorkload"
definition.oam.dev/description: "Long running service with ports exposed"
spec:
definitionRef:
name: podspecworkloads.standard.oam.dev
childResourceKinds:
- apiVersion: apps/v1
kind: Deployment
- apiVersion: v1
kind: Service
---
apiVersion: core.oam.dev/v1alpha2
kind: WorkloadDefinition
metadata:
name: deployment
labels:
workload.oam.dev/podspecable: "true"
spec:
definitionRef:
name: deployments.apps
---
apiVersion: core.oam.dev/v1alpha2
kind: WorkloadDefinition
metadata:
name: deploy
spec:
podSpecPath: spec.template.spec
definitionRef:
name: deployments.apps
documentation/samples/route-demo/deploy.yaml
+60
View File
@@ -0,0 +1,60 @@
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: oam-env-default
spec:
selfSigned: {}
---
apiVersion: core.oam.dev/v1alpha2
kind: Component
metadata:
name: mycomp
spec:
workload:
apiVersion: apps/v1
kind: Deployment
metadata:
name: mycomp
labels:
workload.oam.dev/type: deploy
component.oam.dev/name: mycomp
spec:
replicas: 1
selector:
matchLabels:
component.oam.dev/name: mycomp
template:
metadata:
labels:
component.oam.dev/name: mycomp
spec:
containers:
- image: crccheck/hello-world
imagePullPolicy: Always
name: mycomp
ports:
- containerPort: 8000
name: default
protocol: TCP
---
apiVersion: core.oam.dev/v1alpha2
kind: ApplicationConfiguration
metadata:
name: myapp
spec:
components:
- componentName: mycomp
traits:
- trait:
apiVersion: standard.oam.dev/v1alpha1
kind: Route
metadata:
labels:
trait.oam.dev/type: route
spec:
host: mycomp.mytest.com
# backend:
# port: 8000
tls:
issuerName: oam-env-default
documentation/samples/route-demo/deployment-app.yaml
+58
View File
@@ -0,0 +1,58 @@
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: oam-env-default
spec:
selfSigned: {}
---
apiVersion: core.oam.dev/v1alpha2
kind: Component
metadata:
name: mycomp
spec:
workload:
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
workload.oam.dev/type: webservice
spec:
replicas: 1
selector:
matchLabels:
component.oam.dev/name: mycomp
template:
metadata:
labels:
component.oam.dev/name: mycomp
spec:
containers:
- image: crccheck/hello-world
imagePullPolicy: Always
name: mycomp
ports:
- containerPort: 8000
name: default
protocol: TCP
---
apiVersion: core.oam.dev/v1alpha2
kind: ApplicationConfiguration
metadata:
name: myapp
spec:
components:
- componentName: mycomp
traits:
- trait:
apiVersion: standard.oam.dev/v1alpha1
kind: Route
metadata:
labels:
trait.oam.dev/type: route
spec:
host: mycomp.mytest.com
tls:
issuerName: oam-env-default
documentation/samples/route-demo/webservice-app.yaml
+51
View File
@@ -0,0 +1,51 @@
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: oam-env-default
spec:
selfSigned: {}
---
apiVersion: core.oam.dev/v1alpha2
kind: Component
metadata:
name: mycomp
spec:
workload:
apiVersion: standard.oam.dev/v1alpha1
kind: PodSpecWorkload
metadata:
name: mycomp
labels:
workload.oam.dev/type: webservice
spec:
podSpec:
containers:
- image: crccheck/hello-world
name: mycomp
ports:
- containerPort: 8000
name: default
protocol: TCP
replicas: 1
---
apiVersion: core.oam.dev/v1alpha2
kind: ApplicationConfiguration
metadata:
name: myapp
spec:
components:
- componentName: mycomp
traits:
- trait:
apiVersion: standard.oam.dev/v1alpha1
kind: Route
metadata:
labels:
trait.oam.dev/type: route
spec:
host: mycomp.mytest.com
# backend:
# port: 8000
tls:
issuerName: oam-env-default
go.mod
+1 -1
View File
@@ -9,7 +9,7 @@ require (
github.com/briandowns/spinner v1.11.1
github.com/coreos/prometheus-operator v0.41.1
github.com/crossplane/crossplane-runtime v0.9.0
github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1
github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1.0.20201015120208-c65ccab4f9c1
github.com/fatih/color v1.9.0
github.com/gertd/go-pluralize v0.1.7
github.com/ghodss/yaml v1.0.0
go.sum
+2
View File
@@ -281,6 +281,8 @@ github.com/crossplane/crossplane-runtime v0.9.0/go.mod h1:gNY/21MLBaz5KNP7hmfXbB
github.com/crossplane/crossplane-tools v0.0.0-20200219001116-bb8b2ce46330/go.mod h1:C735A9X0x0lR8iGVOOxb49Mt70Ua4EM2b7PGaRPBLd4=
github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1 h1:N9999ECMaJYf+yhiNgAXk2HIgMeT9E5M7Y1AcXut8dU=
github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1/go.mod h1:D+MDS5vrJZWEA5cxr5kyzCSRQwrt1hLD3ONgC7sVMmc=
github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1.0.20201015120208-c65ccab4f9c1 h1:uAnGnossZ4biy97EKG6xj9zcGD94Uo1kwbdEPR9lZag=
github.com/crossplane/oam-kubernetes-runtime v0.3.0-rc1.0.20201015120208-c65ccab4f9c1/go.mod h1:D+MDS5vrJZWEA5cxr5kyzCSRQwrt1hLD3ONgC7sVMmc=
github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
github.com/daixiang0/gci v0.0.0-20200727065011-66f1df783cb2/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
pkg/application/app.go
+6 -6
View File
@@ -327,7 +327,7 @@ func (app *Application) GetComponentTraits(componentName string, env *types.EnvM
return nil, err
}
//TODO(wonderflow): handle trait data input/output here
obj.SetAnnotations(map[string]string{types.AnnTraitDef: traitType})
obj.SetLabels(map[string]string{oam.TraitTypeLabel: traitType})
traits = append(traits, v1alpha2.ComponentTrait{Trait: runtime.RawExtension{Object: obj}})
}
return traits, nil
@@ -369,13 +369,13 @@ func (app *Application) OAM(env *types.EnvMeta) ([]v1alpha2.Component, v1alpha2.
if err != nil {
return nil, v1alpha2.ApplicationConfiguration{}, nil, err
}
anns := component.Annotations
if anns == nil {
anns = map[string]string{types.AnnWorkloadDef: workloadType}
labels := obj.GetLabels()
if labels == nil {
labels = map[string]string{oam.WorkloadTypeLabel: workloadType}
} else {
anns[types.AnnWorkloadDef] = workloadType
labels[oam.WorkloadTypeLabel] = workloadType
}
component.Annotations = anns
obj.SetLabels(labels)
component.Spec.Workload.Object = obj
components = append(components, component)
pkg/commands/ls.go
+2 -1
View File
@@ -10,6 +10,7 @@ import (
"github.com/spf13/cobra"
"sigs.k8s.io/controller-runtime/pkg/client"
runtimeoam "github.com/crossplane/oam-kubernetes-runtime/pkg/oam"
"github.com/oam-dev/kubevela/api/types"
"github.com/oam-dev/kubevela/pkg/application"
cmdutil "github.com/oam-dev/kubevela/pkg/commands/util"
@@ -132,7 +133,7 @@ func mergeStagingComponents(deployed []apis.ComponentMeta, env *types.EnvMeta, i
all = append(all, apis.ComponentMeta{
Name: c.Name,
App: app.Name,
WorkloadName: c.Annotations[types.AnnWorkloadDef],
WorkloadName: c.Labels[runtimeoam.WorkloadTypeLabel],
TraitNames: traits,
Status: types.StatusStaging,
CreatedTime: app.CreateTime.String(),
pkg/controller/v1alpha1/routes/ingress/api.go
+26
View File
@@ -0,0 +1,26 @@
package ingress
import (
"fmt"
"k8s.io/api/networking/v1beta1"
standardv1alpha1 "github.com/oam-dev/kubevela/api/v1alpha1"
)
const TypeNginx = "nginx"
type RouteIngress interface {
Construct(routeTrait *standardv1alpha1.Route) []*v1beta1.Ingress
}
func GetRouteIngress(provider string) (RouteIngress, error) {
var routeIngress RouteIngress
switch provider {
case TypeNginx, "":
routeIngress = &Nginx{}
default:
return nil, fmt.Errorf("unknow route ingress provider '%v', only '%s' is supported now", provider, TypeNginx)
}
return routeIngress, nil
}
pkg/controller/v1alpha1/routes/ingress/api_test.go
+16
View File
@@ -0,0 +1,16 @@
package ingress
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetRouteIngress(t *testing.T) {
_, err := GetRouteIngress("nginx")
assert.NoError(t, err)
_, err = GetRouteIngress("")
assert.NoError(t, err)
_, err = GetRouteIngress("istio")
assert.EqualError(t, err, "unknow route ingress provider 'istio', only 'nginx' is supported now")
}
pkg/controller/v1alpha1/routes/ingress/nginx_ingress.go
+124
View File
@@ -0,0 +1,124 @@
package ingress
import (
"fmt"
"reflect"
"strconv"
standardv1alpha1 "github.com/oam-dev/kubevela/api/v1alpha1"
v1 "k8s.io/api/core/v1"
"k8s.io/api/networking/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
)
type Nginx struct{}
var _ RouteIngress = &Nginx{}
func (*Nginx) Construct(routeTrait *standardv1alpha1.Route) []*v1beta1.Ingress {
var ingresses []*v1beta1.Ingress
for idx, rule := range routeTrait.Spec.Rules {
name := rule.Name
if name == "" {
name = strconv.Itoa(idx)
}
backend := rule.Backend
if backend == nil || backend.BackendService == nil {
continue
}
var annotations = make(map[string]string)
annotations["kubernetes.io/ingress.class"] = TypeNginx
// SSL
var issuerAnn = "cert-manager.io/issuer"
if routeTrait.Spec.TLS.Type == standardv1alpha1.ClusterIssuer {
issuerAnn = "cert-manager.io/cluster-issuer"
}
annotations[issuerAnn] = routeTrait.Spec.TLS.IssuerName
// Rewrite
if rule.RewriteTarget != "" {
annotations["ingress.kubernetes.io/rewrite-target"] = rule.RewriteTarget
}
// Custom headers
var headerSnippet string
for k, v := range rule.CustomHeaders {
headerSnippet += fmt.Sprintf("more_set_headers \"%s: %s\";\n", k, v)
}
if headerSnippet != "" {
annotations["nginx.ingress.kubernetes.io/configuration-snippet"] = headerSnippet
}
//Send timeout
if backend.SendTimeout != 0 {
annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = strconv.Itoa(backend.SendTimeout)
}
//Read timeout
if backend.ReadTimeout != 0 {
annotations["nginx.ingress.kubernetes.io/proxy‑read‑timeout"] = strconv.Itoa(backend.ReadTimeout)
}
ingress := &v1beta1.Ingress{
TypeMeta: metav1.TypeMeta{
Kind: reflect.TypeOf(v1beta1.Ingress{}).Name(),
APIVersion: v1beta1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: routeTrait.Name + "-" + name,
Namespace: routeTrait.Namespace,
Annotations: annotations,
Labels: routeTrait.GetLabels(),
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: routeTrait.GetObjectKind().GroupVersionKind().GroupVersion().String(),
Kind: routeTrait.GetObjectKind().GroupVersionKind().Kind,
UID: routeTrait.GetUID(),
Name: routeTrait.GetName(),
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
},
},
},
}
ingress.Spec.TLS = []v1beta1.IngressTLS{
{
Hosts: []string{routeTrait.Spec.Host},
SecretName: routeTrait.Name + "-" + name + "-cert",
},
}
if rule.DefaultBackend != nil {
ingress.Spec.Backend = &v1beta1.IngressBackend{
Resource: &v1.TypedLocalObjectReference{
APIGroup: &rule.DefaultBackend.APIVersion,
Kind: rule.DefaultBackend.Kind,
Name: rule.DefaultBackend.Name,
},
}
}
ingress.Spec.Rules = []v1beta1.IngressRule{
{
Host: routeTrait.Spec.Host,
IngressRuleValue: v1beta1.IngressRuleValue{HTTP: &v1beta1.HTTPIngressRuleValue{
Paths: []v1beta1.HTTPIngressPath{
{
Path: rule.Path,
Backend: v1beta1.IngressBackend{
ServiceName: backend.BackendService.ServiceName,
ServicePort: backend.BackendService.Port,
},
},
},
}},
},
}
ingresses = append(ingresses, ingress)
}
return ingresses
}
pkg/controller/v1alpha1/routes/ingress/nginx_ingress_test.go
+109
View File
@@ -0,0 +1,109 @@
package ingress
import (
"strconv"
"testing"
"github.com/crossplane/crossplane-runtime/apis/core/v1alpha1"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
"k8s.io/api/networking/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/pointer"
standardv1alpha1 "github.com/oam-dev/kubevela/api/v1alpha1"
)
func TestConstruct(t *testing.T) {
tests := map[string]struct {
routeTrait *standardv1alpha1.Route
exp []*v1beta1.Ingress
}{
"normal case": {
routeTrait: &standardv1alpha1.Route{
TypeMeta: metav1.TypeMeta{
Kind: "Route",
APIVersion: "standard.oam.dev/v1alpha1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "trait-test",
},
Spec: standardv1alpha1.RouteSpec{
Host: "test.abc",
TLS: &standardv1alpha1.TLS{
IssuerName: "test-issuer",
Type: "Issuer",
},
Rules: []standardv1alpha1.Rule{
{
Name: "myrule1",
Backend: &standardv1alpha1.Backend{BackendService: &standardv1alpha1.BackendServiceRef{ServiceName: "test", Port: intstr.FromInt(3030)}},
DefaultBackend: &v1alpha1.TypedReference{
APIVersion: "k8s.example.com/v1",
Kind: "StorageBucket",
Name: "static-assets",
},
},
},
},
},
exp: []*v1beta1.Ingress{
{
TypeMeta: metav1.TypeMeta{
Kind: "Ingress",
APIVersion: "networking.k8s.io/v1beta1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "trait-test-myrule1",
Annotations: map[string]string{
"kubernetes.io/ingress.class": "nginx",
"cert-manager.io/issuer": "test-issuer",
},
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "standard.oam.dev/v1alpha1",
Kind: "Route",
Name: "trait-test",
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
},
},
},
Spec: v1beta1.IngressSpec{
TLS: []v1beta1.IngressTLS{
{
Hosts: []string{"test.abc"},
SecretName: "trait-test-myrule1-cert",
},
},
Backend: &v1beta1.IngressBackend{Resource: &v1.TypedLocalObjectReference{
APIGroup: pointer.StringPtr("k8s.example.com/v1"),
Kind: "StorageBucket",
Name: "static-assets",
}},
Rules: []v1beta1.IngressRule{
{
Host: "test.abc",
IngressRuleValue: v1beta1.IngressRuleValue{HTTP: &v1beta1.HTTPIngressRuleValue{Paths: []v1beta1.HTTPIngressPath{
{
Path: "",
Backend: v1beta1.IngressBackend{ServiceName: "test", ServicePort: intstr.FromInt(3030)},
},
}}},
},
},
},
},
},
},
}
for message, ti := range tests {
nginx := &Nginx{}
got := nginx.Construct(ti.routeTrait)
assert.Equal(t, len(ti.exp), len(got))
for idx := range ti.exp {
assert.Equal(t, ti.exp[idx], got[idx], message+" index "+strconv.Itoa(idx))
}
}
}
pkg/controller/v1alpha1/routes/route_controller.go
+138 -250
View File
@@ -21,11 +21,11 @@ import (
"encoding/json"
"fmt"
"reflect"
"strconv"
"github.com/oam-dev/kubevela/api/v1alpha1"
standardv1alpha1 "github.com/oam-dev/kubevela/api/v1alpha1"
"github.com/oam-dev/kubevela/pkg/controller/common"
"github.com/oam-dev/kubevela/pkg/controller/v1alpha1/routes/ingress"
cpv1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1"
runtimev1alpha1 "github.com/crossplane/crossplane-runtime/apis/core/v1alpha1"
@@ -51,15 +51,6 @@ const (
errCreateIssuer = "failed to create cert-manager Issuer"
)
var (
// oamServiceLabel is the pre-defined labels for any serviceMonitor
// created by the RouteTrait
oamServiceLabel = map[string]string{
"k8s-app": "oam",
"controller": "routeTrait",
}
)
// Reconciler reconciles a Route object
type Reconciler struct {
client.Client
@@ -70,7 +61,6 @@ type Reconciler struct {
// +kubebuilder:rbac:groups=standard.oam.dev,resources=routes,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=standard.oam.dev,resources=routes/status,verbs=get;update;patch
func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
ctx := context.Background()
mLog := r.Log.WithValues("route", req.NamespacedName)
@@ -83,7 +73,6 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
}
mLog.Info("Get the route trait",
"host", routeTrait.Spec.Host,
"path", routeTrait.Spec.Path,
"workload reference", routeTrait.Spec.WorkloadReference,
"labels", routeTrait.GetLabels())
@@ -105,35 +94,15 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
oamutil.PatchCondition(ctx, r, &routeTrait,
cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrLocatingWorkload)))
}
// try to see if the workload already has services as child resources, and match for our route
svc, exist, svcPort, err := r.fetchService(ctx, mLog, workload, &routeTrait)
if err != nil && !apierrors.IsNotFound(err) {
r.record.Event(eventObj, event.Warning(common.ErrLocatingService, err))
return oamutil.ReconcileWaitResult,
oamutil.PatchCondition(ctx, r, &routeTrait,
cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrLocatingService)))
}
// Create Services
if !exist {
// no service found, we will create service according to rule
svc, svcPort, err = r.createService(ctx, mLog, workload, &routeTrait)
if err != nil {
r.record.Event(eventObj, event.Warning(common.ErrCreatingService, err))
return oamutil.ReconcileWaitResult,
oamutil.PatchCondition(ctx, r, &routeTrait,
cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrCreatingService)))
var svc *runtimev1alpha1.TypedReference
if NeedDiscovery(&routeTrait) {
if svc, err = r.discoveryAndFillBackend(ctx, mLog, eventObj, workload, &routeTrait); err != nil {
return oamutil.ReconcileWaitResult, err
}
r.record.Event(eventObj, event.Normal("Service created",
fmt.Sprintf("successfully automatically created a service `%s`", svc.Name)))
} else {
mLog.Info("workload already has service as child resource, will not create new", "workloadName", workload.GetName())
}
// Create Issuers
var issuer standardv1alpha1.TLS
if routeTrait.Spec.TLS == nil || routeTrait.Spec.TLS.IssuerName == "" {
// Create Issuer
if routeTrait.Spec.TLS == nil {
issuerName, err := r.createSelfsignedIssuer(ctx, &routeTrait)
if err != nil {
r.record.Event(eventObj, event.Warning(errCreateIssuer, err))
@@ -143,42 +112,93 @@ func (r *Reconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
}
r.record.Event(eventObj, event.Normal("Issuer created",
fmt.Sprintf("successfully automatically created a Issuer for route TLS `%s`", issuerName)))
issuer.Type = standardv1alpha1.NamespaceIssuer
issuer.IssuerName = issuerName
} else {
issuer = *routeTrait.Spec.TLS
// All rules will use the same selfsigned issuer.
routeTrait.Spec.TLS = &v1alpha1.TLS{
IssuerName: issuerName,
Type: standardv1alpha1.NamespaceIssuer,
}
}
ingressConstructer, err := ingress.GetRouteIngress(routeTrait.Spec.Provider)
if err != nil {
mLog.Error(err, "Failed to get routeIngress, use nginx route instead")
ingressConstructer = &ingress.Nginx{}
}
// Create Ingress
// construct the serviceMonitor that hooks the service to the prometheus server
ingress := constructNginxIngress(&routeTrait, issuer, svc, svcPort)
ingresses := ingressConstructer.Construct(&routeTrait)
// server side apply the serviceMonitor, only the fields we set are touched
applyOpts := []client.PatchOption{client.ForceOwnership, client.FieldOwner(routeTrait.GetUID())}
if err := r.Patch(ctx, ingress, client.Apply, applyOpts...); err != nil {
mLog.Error(err, "Failed to apply to ingress")
r.record.Event(eventObj, event.Warning(errApplyNginxIngress, err))
return oamutil.ReconcileWaitResult,
oamutil.PatchCondition(ctx, r, &routeTrait,
cpv1alpha1.ReconcileError(errors.Wrap(err, errApplyNginxIngress)))
for _, ingress := range ingresses {
if err := r.Patch(ctx, ingress, client.Apply, applyOpts...); err != nil {
mLog.Error(err, "Failed to apply to ingress")
r.record.Event(eventObj, event.Warning(errApplyNginxIngress, err))
return oamutil.ReconcileWaitResult,
oamutil.PatchCondition(ctx, r, &routeTrait,
cpv1alpha1.ReconcileError(errors.Wrap(err, errApplyNginxIngress)))
}
r.record.Event(eventObj, event.Normal("Nginx Ingress created",
fmt.Sprintf("successfully server side patched a route trait `%s`", routeTrait.Name)))
}
r.record.Event(eventObj, event.Normal("Nginx Ingress created",
fmt.Sprintf("successfully server side patched a route trait `%s`", routeTrait.Name)))
// TODO(wonderflow): GC mechanism for no used ingress, service, issuer
routeTrait.Status.Service = svc
routeTrait.Status.Ingress = &runtimev1alpha1.TypedReference{
APIVersion: v1beta1.SchemeGroupVersion.String(),
Kind: reflect.TypeOf(v1beta1.Ingress{}).Name(),
Name: ingress.Name,
UID: routeTrait.UID,
var ingressCreated []runtimev1alpha1.TypedReference
for _, ingress := range ingresses {
ingressCreated = append(ingressCreated, runtimev1alpha1.TypedReference{
APIVersion: v1beta1.SchemeGroupVersion.String(),
Kind: reflect.TypeOf(v1beta1.Ingress{}).Name(),
Name: ingress.Name,
UID: routeTrait.UID,
})
}
return ctrl.Result{}, oamutil.PatchCondition(ctx, r, &routeTrait)
routeTrait.Status.Ingresses = ingressCreated
routeTrait.Status.Service = svc
return ctrl.Result{}, r.Status().Update(ctx, &routeTrait)
}
// create a service that targets the exposed workload pod
func (r *Reconciler) createService(ctx context.Context, mLog logr.Logger, workload *unstructured.Unstructured,
routeTrait *v1alpha1.Route) (*runtimev1alpha1.TypedReference, int32, error) {
// discoveryAndFillBackend will automatically discovery backend for route
func (r *Reconciler) discoveryAndFillBackend(ctx context.Context, mLog logr.Logger, eventObj runtime.Object, workload *unstructured.Unstructured,
routeTrait *v1alpha1.Route) (*runtimev1alpha1.TypedReference, error) {
// Fetch the child childResources list from the corresponding workload
childResources, err := oamutil.FetchWorkloadChildResources(ctx, mLog, r, workload)
if err != nil {
mLog.Error(err, "Error while fetching the workload child childResources", "workload kind", workload.GetKind(),
"workload name", workload.GetName())
if !apierrors.IsNotFound(err) {
return nil, err
}
}
// try to see if the workload already has services in child childResources, and match for our route
err = r.fillBackendByCheckChildResource(mLog, routeTrait, childResources)
if err != nil && !apierrors.IsNotFound(err) {
r.record.Event(eventObj, event.Warning(common.ErrLocatingService, err))
return nil, oamutil.PatchCondition(ctx, r, routeTrait,
cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrLocatingService)))
}
// Check if still need discovery after childResource filled.
if NeedDiscovery(routeTrait) {
// no service found, we will create service according to rule
svc, err := r.fillBackendByCreatedService(ctx, mLog, workload, routeTrait, childResources)
if err != nil {
r.record.Event(eventObj, event.Warning(common.ErrCreatingService, err))
return nil, oamutil.PatchCondition(ctx, r, routeTrait,
cpv1alpha1.ReconcileError(errors.Wrap(err, common.ErrCreatingService)))
}
r.record.Event(eventObj, event.Normal("Service created",
fmt.Sprintf("successfully automatically created a service `%s`", svc.Name)))
return svc, nil
}
mLog.Info("workload already has service as child resource, will not create service", "workloadName", workload.GetName())
return nil, nil
}
// fillBackendByCreatedService will automatically create service by discovery podTemplate or podSpec.
func (r *Reconciler) fillBackendByCreatedService(ctx context.Context, mLog logr.Logger, workload *unstructured.Unstructured,
routeTrait *v1alpha1.Route, childResources []*unstructured.Unstructured) (*runtimev1alpha1.TypedReference, error) {
oamService := &corev1.Service{
TypeMeta: metav1.TypeMeta{
@@ -186,9 +206,9 @@ func (r *Reconciler) createService(ctx context.Context, mLog logr.Logger, worklo
APIVersion: common.ServiceAPIVersion,
},
ObjectMeta: metav1.ObjectMeta{
Name: "route-" + workload.GetName(),
Namespace: workload.GetNamespace(),
Labels: oamServiceLabel,
Name: routeTrait.GetName(),
Namespace: routeTrait.GetNamespace(),
Labels: filterLabels(routeTrait.GetLabels()),
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: routeTrait.GetObjectKind().GroupVersionKind().GroupVersion().String(),
@@ -204,97 +224,70 @@ func (r *Reconciler) createService(ctx context.Context, mLog logr.Logger, worklo
Type: corev1.ServiceTypeClusterIP,
},
}
// assign selector
if routeTrait.Spec.Backend != nil && len(routeTrait.Spec.Backend.SelectLabels) != 0 {
oamService.Spec.Selector = routeTrait.Spec.Backend.SelectLabels
}
port, labels, err := DiscoverPortLabel(ctx, mLog, workload, r)
if err == nil {
if len(oamService.Spec.Selector) == 0 {
oamService.Spec.Selector = labels
}
if routeTrait.Spec.Backend == nil {
routeTrait.Spec.Backend = &standardv1alpha1.Backend{Port: port}
} else if routeTrait.Spec.Backend.Port.String() == "0" {
routeTrait.Spec.Backend.Port = port
}
} else {
ports, labels, err := DiscoverPortsLabel(ctx, workload, r, childResources)
if err != nil {
mLog.Info("[WARN] fail to discovery port and label", "err", err)
return nil, err
}
oamService.Spec.Selector = labels
var servicePort int32 = 443
oamService.Spec.Ports = []corev1.ServicePort{
{
Port: servicePort,
TargetPort: routeTrait.Spec.Backend.Port,
// use the same port
for _, port := range ports {
oamService.Spec.Ports = append(oamService.Spec.Ports, corev1.ServicePort{
Port: int32(port.IntValue()),
TargetPort: port,
Protocol: corev1.ProtocolTCP,
},
})
}
// server side apply the service, only the fields we set are touched
applyOpts := []client.PatchOption{client.ForceOwnership, client.FieldOwner(routeTrait.GetUID())}
if err := r.Patch(ctx, oamService, client.Apply, applyOpts...); err != nil {
mLog.Error(err, "Failed to apply to service")
return nil, servicePort, err
return nil, err
}
FillRouteTraitWithService(oamService, routeTrait)
return &runtimev1alpha1.TypedReference{
APIVersion: common.ServiceAPIVersion,
Kind: common.ServiceKind,
Name: oamService.Name,
UID: routeTrait.UID,
}, servicePort, nil
}, nil
}
// Assume the workload or it's childResource will always having spec.template as PodTemplate if discoverable
func DiscoverPortLabel(ctx context.Context, mLog logr.Logger, workload *unstructured.Unstructured, r client.Reader) (intstr.IntOrString, map[string]string, error) {
var resources = []*unstructured.Unstructured{workload}
// Fetch the child resources list from the corresponding workload
childResources, err := oamutil.FetchWorkloadChildResources(ctx, mLog, r, workload)
if err == nil {
resources = append(resources, childResources...)
} else {
mLog.Info("[WARN] fail to fetch workload child resource", "name", workload.GetName(), "err", err)
func DiscoverPortsLabel(ctx context.Context, workload *unstructured.Unstructured, r client.Reader, childResources []*unstructured.Unstructured) ([]intstr.IntOrString, map[string]string, error) {
// here is the logic follows the design https://github.com/crossplane/oam-kubernetes-runtime/blob/master/design/one-pager-podspecable-workload.md#proposal
// Get WorkloadDefinition
workloadDef, err := oamutil.FetchWorkloadDefinition(ctx, r, workload)
if err != nil {
return nil, nil, err
}
podSpecPath, ok := GetPodSpecPath(workloadDef)
if podSpecPath != "" {
ports, err := discoveryFromPodSpec(workload, podSpecPath)
if err != nil {
return nil, nil, err
}
return ports, filterLabels(workload.GetLabels()), nil
}
if ok {
return discoveryFromPodTemplate(workload, "spec", "template")
}
// If workload is not podSpecable, try to detect it's child resource
var resources = []*unstructured.Unstructured{workload}
resources = append(resources, childResources...)
var gatherErrs []error
for _, w := range resources {
port, labels, err := discoveryFromObject(w)
port, labels, err := discoveryFromPodTemplate(w, "spec", "template")
if err == nil {
return port, labels, nil
}
gatherErrs = append(gatherErrs, err)
}
return intstr.IntOrString{}, nil, fmt.Errorf("can't discovery port from workload %v %v.%v and it's child resource, errorList: %v", workload.GetName(), workload.GetAPIVersion(), workload.GetKind(), gatherErrs)
}
func discoveryFromObject(w *unstructured.Unstructured) (intstr.IntOrString, map[string]string, error) {
obj, found, _ := unstructured.NestedMap(w.Object, "spec", "template")
if !found {
return intstr.IntOrString{}, nil, fmt.Errorf("not have spec.template in workload %v", w.GetName())
}
data, err := json.Marshal(obj)
if err != nil {
return intstr.IntOrString{}, nil, fmt.Errorf("workload %v convert object err %v", w.GetName(), err)
}
var template corev1.PodTemplate
err = json.Unmarshal(data, &template)
if err != nil {
return intstr.IntOrString{}, nil, fmt.Errorf("workload %v convert object to PodTemplate err %v", w.GetName(), err)
}
port := getFirstPort(template.Template.Spec.Containers)
if port == 0 {
return intstr.IntOrString{}, nil, fmt.Errorf("no port found in workload %v", w.GetName())
}
return intstr.FromInt(int(port)), template.Labels, nil
}
func getFirstPort(cs []corev1.Container) int32 {
//TODO(wonderflow): exclude some sidecars
for _, container := range cs {
for _, p := range container.Ports {
return p.ContainerPort
}
}
return 0
return nil, nil, fmt.Errorf("fail to automatically discovery backend from workload %v(%v.%v) and it's child resource, errorList: %v", workload.GetName(), workload.GetAPIVersion(), workload.GetKind(), gatherErrs)
}
func (r *Reconciler) createSelfsignedIssuer(ctx context.Context, routeTrait *v1alpha1.Route) (string, error) {
@@ -320,135 +313,30 @@ func (r *Reconciler) createSelfsignedIssuer(ctx context.Context, routeTrait *v1a
return selfSigned, fmt.Errorf("get %s err %v", selfSigned, err)
}
func constructNginxIngress(routeTrait *standardv1alpha1.Route, issuer standardv1alpha1.TLS, service *runtimev1alpha1.TypedReference, port int32) *v1beta1.Ingress {
var annotations = make(map[string]string)
// Use nginx-ingress as implementation
annotations["kubernetes.io/ingress.class"] = "nginx"
// SSL
var issuerAnn = "cert-manager.io/issuer"
if issuer.Type == standardv1alpha1.ClusterIssuer {
issuerAnn = "cert-manager.io/cluster-issuer"
}
annotations[issuerAnn] = issuer.IssuerName
// Rewrite
if routeTrait.Spec.RewriteTarget != "" {
annotations["ingress.kubernetes.io/rewrite-target"] = routeTrait.Spec.RewriteTarget
}
// Custom headers
var headerSnippet string
for k, v := range routeTrait.Spec.CustomHeaders {
headerSnippet += fmt.Sprintf("more_set_headers \"%s: %s\";\n", k, v)
}
if headerSnippet != "" {
annotations["nginx.ingress.kubernetes.io/configuration-snippet"] = headerSnippet
}
backend := routeTrait.Spec.Backend
if backend != nil {
// Backend protocol
if backend.Protocol != "" {
annotations["nginx.ingress.kubernetes.io/backend-protocol"] = backend.Protocol
}
//Send timeout
if backend.SendTimeout != 0 {
annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = strconv.Itoa(backend.SendTimeout)
}
//Read timeout
if backend.ReadTimeout != 0 {
annotations["nginx.ingress.kubernetes.io/proxy‑read‑timeout"] = strconv.Itoa(backend.ReadTimeout)
}
}
ingress := &v1beta1.Ingress{
TypeMeta: metav1.TypeMeta{
Kind: reflect.TypeOf(v1beta1.Ingress{}).Name(),
APIVersion: v1beta1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: routeTrait.Name,
Namespace: routeTrait.Namespace,
Annotations: annotations,
Labels: oamServiceLabel,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: routeTrait.GetObjectKind().GroupVersionKind().GroupVersion().String(),
Kind: routeTrait.GetObjectKind().GroupVersionKind().Kind,
UID: routeTrait.GetUID(),
Name: routeTrait.GetName(),
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
},
},
},
}
ingress.Spec.TLS = []v1beta1.IngressTLS{
{
Hosts: []string{routeTrait.Spec.Host},
SecretName: routeTrait.Name + "-cert",
},
}
if routeTrait.Spec.DefaultBackend != nil {
ingress.Spec.Backend = routeTrait.Spec.DefaultBackend
}
ingress.Spec.Rules = []v1beta1.IngressRule{
{
Host: routeTrait.Spec.Host,
IngressRuleValue: v1beta1.IngressRuleValue{HTTP: &v1beta1.HTTPIngressRuleValue{
Paths: []v1beta1.HTTPIngressPath{
{
Path: routeTrait.Spec.Path,
Backend: v1beta1.IngressBackend{
ServiceName: service.Name,
ServicePort: intstr.FromInt(int(port)),
},
},
},
}},
},
}
return ingress
}
// fetch the service that is associated with the workload
func (r *Reconciler) fetchService(ctx context.Context, mLog logr.Logger,
workload *unstructured.Unstructured, routeTrait *v1alpha1.Route) (*runtimev1alpha1.TypedReference, bool, int32, error) {
// Fetch the child resources list from the corresponding workload
resources, err := oamutil.FetchWorkloadChildResources(ctx, mLog, r, workload)
if err != nil {
if !apierrors.IsNotFound(err) {
mLog.Error(err, "Error while fetching the workload child resources", "workload kind", workload.GetKind(),
"workload name", workload.GetName())
}
return nil, false, 0, err
func (r *Reconciler) fillBackendByCheckChildResource(mLog logr.Logger,
routeTrait *v1alpha1.Route, childResources []*unstructured.Unstructured) error {
if len(childResources) == 0 {
return nil
}
// find the service that has the port
for _, childRes := range resources {
for _, childRes := range childResources {
if childRes.GetAPIVersion() == corev1.SchemeGroupVersion.String() && childRes.GetKind() == reflect.TypeOf(corev1.Service{}).Name() {
svc := &runtimev1alpha1.TypedReference{
APIVersion: common.ServiceAPIVersion,
Kind: common.ServiceKind,
Name: childRes.GetName(),
UID: childRes.GetUID(),
data, err := json.Marshal(childRes.Object)
if err != nil {
mLog.Error(err, "error marshal child childResources as K8s Service, continue to check other resource", "resource name", childRes.GetName())
continue
}
ports, _, _ := unstructured.NestedSlice(childRes.Object, "spec", "ports")
for _, port := range ports {
data, _ := json.Marshal(port)
var servicePort corev1.ServicePort
_ = json.Unmarshal(data, &servicePort)
if routeTrait.Spec.Backend == nil || routeTrait.Spec.Backend.Port.IntValue() == 0 || servicePort.TargetPort == routeTrait.Spec.Backend.Port {
return svc, true, servicePort.Port, nil
}
var service corev1.Service
err = json.Unmarshal(data, &service)
if err != nil {
mLog.Error(err, "error unmarshal child childResources as K8s Service, continue to check other resource", "resource name", childRes.GetName())
continue
}
FillRouteTraitWithService(&service, routeTrait)
}
}
return nil, false, 0, nil
return nil
}
func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
pkg/controller/v1alpha1/routes/route_controller_test.go
+351
View File
@@ -0,0 +1,351 @@
package routes
import (
"context"
"errors"
"time"
"github.com/oam-dev/kubevela/api/v1alpha1"
"github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2"
"github.com/crossplane/oam-kubernetes-runtime/pkg/oam/util"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
certmanager "github.com/wonderflow/cert-manager-api/pkg/apis/certmanager/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/api/networking/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
logf "sigs.k8s.io/controller-runtime/pkg/log"
)
var _ = Describe("Route Trait Integration Test", func() {
// common var init
ctx := context.Background()
namespaceName := "routetrait-integration-test"
podPort := 8000
issuerName := "my-issuer"
var ns corev1.Namespace
getComponent := func(workloadType, compName string) (v1alpha2.Component, map[string]string, map[string]string) {
podTemplateLabel := map[string]string{"standard.oam.dev": "oam-test-deployment", "workload.oam.dev/type": workloadType}
workloadLabel := map[string]string{"standard.oam.dev": "oam-test-deployment", "app.oam.dev/component": compName, "app.oam.dev/name": "test-app-" + compName}
basedeploy := &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Labels: podTemplateLabel,
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: podTemplateLabel,
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: podTemplateLabel,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "container-name",
Image: "crccheck/hello-world",
ImagePullPolicy: corev1.PullNever,
Ports: []corev1.ContainerPort{
{
ContainerPort: int32(podPort),
}}}}}}},
}
var rp = int32(1)
if workloadType == "webservice" {
basePodSpecc := &v1alpha1.PodSpecWorkload{
TypeMeta: metav1.TypeMeta{
Kind: "PodSpecWorkload",
APIVersion: "standard.oam.dev/v1alpha1",
},
ObjectMeta: metav1.ObjectMeta{
Labels: podTemplateLabel,
},
Spec: v1alpha1.PodSpecWorkloadSpec{
Replicas: &rp,
PodSpec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "container-name",
Image: "crccheck/hello-world",
ImagePullPolicy: corev1.PullNever,
Ports: []corev1.ContainerPort{
{
ContainerPort: int32(podPort),
}}}}}},
}
return v1alpha2.Component{
TypeMeta: metav1.TypeMeta{
Kind: "Component",
APIVersion: "core.oam.dev/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: compName,
Namespace: ns.Name,
},
Spec: v1alpha2.ComponentSpec{
Workload: runtime.RawExtension{Object: basePodSpecc},
},
}, workloadLabel, podTemplateLabel
}
return v1alpha2.Component{
TypeMeta: metav1.TypeMeta{
Kind: "Component",
APIVersion: "core.oam.dev/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: compName,
Namespace: ns.Name,
},
Spec: v1alpha2.ComponentSpec{
Workload: runtime.RawExtension{Object: basedeploy},
},
}, workloadLabel, podTemplateLabel
}
getAC := func(compName string) v1alpha2.ApplicationConfiguration {
return v1alpha2.ApplicationConfiguration{
TypeMeta: metav1.TypeMeta{
Kind: "ApplicationConfiguration",
APIVersion: "core.oam.dev/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: ns.Name,
Name: "test-app-" + compName,
},
Spec: v1alpha2.ApplicationConfigurationSpec{
Components: []v1alpha2.ApplicationConfigurationComponent{{
ComponentName: compName,
Traits: []v1alpha2.ComponentTrait{
{
Trait: runtime.RawExtension{Object: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "standard.oam.dev/v1alpha1",
"kind": "Route",
"metadata": map[string]interface{}{
"labels": map[string]interface{}{
"trait.oam.dev/type": "route",
},
},
"spec": map[string]interface{}{
"host": "mycomp.mytest.com",
"tls": map[string]interface{}{
"issuerName": issuerName,
}}}}}}}}}}}
}
BeforeEach(func() {
logf.Log.Info("[TEST] Set up resources before an integration test")
ns = corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: namespaceName,
},
}
By("Create the Namespace for test")
Expect(k8sClient.Create(ctx, &ns)).Should(SatisfyAny(Succeed(), &util.AlreadyExistMatcher{}))
By("Create the Issuer for test")
Expect(k8sClient.Create(context.Background(), &certmanager.Issuer{
ObjectMeta: metav1.ObjectMeta{Name: issuerName, Namespace: namespaceName},
Spec: certmanager.IssuerSpec{IssuerConfig: certmanager.IssuerConfig{SelfSigned: &certmanager.SelfSignedIssuer{}}},
})).Should(SatisfyAny(Succeed(), &util.AlreadyExistMatcher{}))
})
AfterEach(func() {
// Control-runtime test environment has a bug that can't delete resources like deployment/namespaces
// We have to use different names to segregate between tests
logf.Log.Info("[TEST] Clean up resources after an integration test")
})
It("Test with child resource no podSpecable but has service child using webservice workload", func() {
compName := "test-webservice"
comp, _, _ := getComponent("webservice", compName)
ac := getAC(compName)
Expect(k8sClient.Create(ctx, &comp)).ToNot(HaveOccurred())
Expect(k8sClient.Create(ctx, &ac)).ToNot(HaveOccurred())
By("Check that we have created the route")
createdRoute := v1alpha1.Route{}
var traitName string
Eventually(
func() error {
err := k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: ac.Name},
&ac)
if err != nil {
return err
}
if len(ac.Status.Workloads) < 1 || len(ac.Status.Workloads[0].Traits) < 1 {
return errors.New("workload or trait not ready")
}
traitName = ac.Status.Workloads[0].Traits[0].Reference.Name
err = k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: traitName},
&createdRoute)
if err != nil {
return err
}
if len(createdRoute.Status.Ingresses) == 0 {
return errors.New("no ingress created")
}
return nil
},
time.Second*30, time.Millisecond*500).Should(BeNil())
By("Check that we have created the ingress")
createdIngress := v1beta1.Ingress{}
Eventually(
func() error {
return k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: createdRoute.Status.Ingresses[0].Name},
&createdIngress)
},
time.Second*30, time.Millisecond*500).Should(BeNil())
logf.Log.Info("[TEST] Get the created ingress", "ingress rules", createdIngress.Spec.Rules)
Expect(createdIngress.GetNamespace()).Should(Equal(namespaceName))
Expect(len(createdIngress.Spec.Rules)).Should(Equal(1))
Expect(createdIngress.Spec.Rules[0].Host).Should(Equal("mycomp.mytest.com"))
Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName).Should(Equal(compName))
Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort.IntVal).Should(Equal(int32(8080)))
})
It("Test with podSpec label with no podSpecPath using deployment workload", func() {
compName := "test-deployment"
comp, _, deploylabel := getComponent("deployment", compName)
ac := getAC(compName)
Expect(k8sClient.Create(ctx, &comp)).ToNot(HaveOccurred())
Expect(k8sClient.Create(ctx, &ac)).ToNot(HaveOccurred())
By("Check that we have created the route")
createdRoute := v1alpha1.Route{}
var traitName string
Eventually(
func() error {
err := k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: ac.Name},
&ac)
if err != nil {
return err
}
if len(ac.Status.Workloads) < 1 || len(ac.Status.Workloads[0].Traits) < 1 {
return errors.New("workload or trait not ready")
}
traitName = ac.Status.Workloads[0].Traits[0].Reference.Name
err = k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: traitName},
&createdRoute)
if err != nil {
return err
}
if len(createdRoute.Status.Ingresses) == 0 {
return errors.New("no ingress created")
}
return nil
},
time.Second*30, time.Millisecond*500).Should(BeNil())
By("Check that we have created the ingress")
createdIngress := v1beta1.Ingress{}
Eventually(
func() error {
return k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: createdRoute.Status.Ingresses[0].Name},
&createdIngress)
},
time.Second*30, time.Millisecond*500).Should(BeNil())
logf.Log.Info("[TEST] Get the created ingress", "ingress rules", createdIngress.Spec.Rules)
Expect(createdIngress.GetNamespace()).Should(Equal(namespaceName))
Expect(len(createdIngress.Spec.Rules)).Should(Equal(1))
Expect(createdIngress.Spec.Rules[0].Host).Should(Equal("mycomp.mytest.com"))
Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName).Should(Equal(traitName))
Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort.IntVal).Should(Equal(int32(8000)))
By("Check that we have created the service")
createdSvc := corev1.Service{}
Eventually(
func() error {
return k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: traitName},
&createdSvc)
},
time.Second*30, time.Millisecond*500).Should(BeNil())
logf.Log.Info("[TEST] Get the created service", "service ports", createdSvc.Spec.Ports)
Expect(createdSvc.Spec.Selector).Should(Equal(deploylabel))
Expect(createdSvc.Spec.Ports[0].TargetPort.IntVal).Should(Equal(int32(podPort)))
})
It("Test with podSpecPath specified using deploy workload", func() {
compName := "test-deploy"
comp, workloadLabel, _ := getComponent("deploy", compName)
ac := getAC(compName)
Expect(k8sClient.Create(ctx, &comp)).ToNot(HaveOccurred())
Expect(k8sClient.Create(ctx, &ac)).ToNot(HaveOccurred())
By("Check that we have created the route")
createdRoute := v1alpha1.Route{}
var traitName string
Eventually(
func() error {
err := k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: ac.Name},
&ac)
if err != nil {
return err
}
if len(ac.Status.Workloads) < 1 || len(ac.Status.Workloads[0].Traits) < 1 {
return errors.New("workload or trait not ready")
}
traitName = ac.Status.Workloads[0].Traits[0].Reference.Name
err = k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: traitName},
&createdRoute)
if err != nil {
return err
}
if len(createdRoute.Status.Ingresses) == 0 {
return errors.New("no ingress created")
}
return nil
},
time.Second*30, time.Millisecond*500).Should(BeNil())
By("Check that we have created the ingress")
createdIngress := v1beta1.Ingress{}
Eventually(
func() error {
return k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: createdRoute.Status.Ingresses[0].Name},
&createdIngress)
},
time.Second*30, time.Millisecond*500).Should(BeNil())
logf.Log.Info("[TEST] Get the created ingress", "ingress rules", createdIngress.Spec.Rules)
Expect(createdIngress.GetNamespace()).Should(Equal(namespaceName))
Expect(len(createdIngress.Spec.Rules)).Should(Equal(1))
Expect(createdIngress.Spec.Rules[0].Host).Should(Equal("mycomp.mytest.com"))
Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName).Should(Equal(traitName))
Expect(createdIngress.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort.IntVal).Should(Equal(int32(8000)))
By("Check that we have created the service")
createdSvc := corev1.Service{}
Eventually(
func() error {
return k8sClient.Get(ctx,
types.NamespacedName{Namespace: ns.Name, Name: traitName},
&createdSvc)
},
time.Second*30, time.Millisecond*500).Should(BeNil())
logf.Log.Info("[TEST] Get the created service", "service ports", createdSvc.Spec.Ports)
for k, v := range workloadLabel {
Expect(createdSvc.Spec.Selector).Should(HaveKeyWithValue(k, v))
}
Expect(createdSvc.Spec.Ports[0].TargetPort.IntVal).Should(Equal(int32(podPort)))
})
})
pkg/controller/v1alpha1/routes/suite_test.go
+97 -10
View File
@@ -17,13 +17,27 @@ limitations under the License.
package routes
import (
"context"
"path/filepath"
"testing"
"github.com/oam-dev/kubevela/pkg/controller/v1alpha1/podspecworkload"
"github.com/crossplane/crossplane-runtime/pkg/logging"
"github.com/crossplane/oam-kubernetes-runtime/pkg/controller"
"github.com/crossplane/oam-kubernetes-runtime/pkg/controller/v1alpha2/applicationconfiguration"
oamCore "github.com/crossplane/oam-kubernetes-runtime/apis/core"
"github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
certmanager "github.com/wonderflow/cert-manager-api/pkg/apis/certmanager/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
@@ -40,6 +54,10 @@ import (
var cfg *rest.Config
var k8sClient client.Client
var testEnv *envtest.Environment
var controllerDone chan struct{}
var routeNS corev1.Namespace
var RouteNSName = "route-test"
func TestAPIs(t *testing.T) {
RegisterFailHandler(Fail)
@@ -50,34 +68,103 @@ func TestAPIs(t *testing.T) {
}
var _ = BeforeSuite(func(done Done) {
logf.SetLogger(zap.LoggerTo(GinkgoWriter, true))
By("bootstrapping test environment")
useExistCluster := true
logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)))
routeNS = corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: RouteNSName,
},
}
By("Bootstrapping test environment")
useExistCluster := false
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
CRDDirectoryPaths: []string{
filepath.Join("../../../..", "charts/vela-core/crds"), // this has all the required CRDs,
},
UseExistingCluster: &useExistCluster,
}
var err error
cfg, err = testEnv.Start()
Expect(err).ToNot(HaveOccurred())
Expect(cfg).ToNot(BeNil())
err = standardv1alpha1.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())
Expect(standardv1alpha1.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
Expect(oamCore.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
Expect(certmanager.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred())
// +kubebuilder:scaffold:scheme
By("Create the k8s client")
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
Expect(err).ToNot(HaveOccurred())
Expect(k8sClient).ToNot(BeNil())
By("Starting the route trait controller in the background")
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
Scheme: scheme.Scheme,
Port: 9443,
})
Expect(err).ToNot(HaveOccurred())
r := Reconciler{
Client: mgr.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("RouteTrait"),
Scheme: mgr.GetScheme(),
}
Expect(r.SetupWithManager(mgr)).ToNot(HaveOccurred())
Expect(applicationconfiguration.Setup(mgr, controller.Args{}, logging.NewLogrLogger(ctrl.Log.WithName("AppConfig")))).ToNot(HaveOccurred())
Expect(podspecworkload.Setup(mgr)).ToNot(HaveOccurred())
controllerDone = make(chan struct{}, 1)
// +kubebuilder:scaffold:builder
go func() {
defer GinkgoRecover()
Expect(mgr.Start(controllerDone)).ToNot(HaveOccurred())
}()
By("Create the routeTrait namespace")
Expect(k8sClient.Create(context.Background(), &routeNS)).ToNot(HaveOccurred())
routeDef := &v1alpha2.TraitDefinition{}
routeDef.Name = "route"
routeDef.Namespace = RouteNSName
routeDef.Spec.Reference.Name = "routes.standard.oam.dev"
routeDef.Spec.WorkloadRefPath = "spec.workloadRef"
Expect(k8sClient.Create(context.Background(), routeDef)).ToNot(HaveOccurred())
webservice := &v1alpha2.WorkloadDefinition{}
webservice.Name = "webservice"
webservice.Namespace = RouteNSName
webservice.Spec.Reference.Name = "podspecworkloads.standard.oam.dev"
webservice.Spec.ChildResourceKinds = []v1alpha2.ChildResourceKind{{
APIVersion: "apps/v1",
Kind: "Deployment",
}, {
APIVersion: "v1",
Kind: "Service",
}}
Expect(k8sClient.Create(context.Background(), webservice)).ToNot(HaveOccurred())
deployment := &v1alpha2.WorkloadDefinition{}
deployment.Name = "deployment"
deployment.Namespace = RouteNSName
deployment.Labels = map[string]string{"workload.oam.dev/podspecable": "true"}
deployment.Spec.Reference.Name = "deployments.apps"
Expect(k8sClient.Create(context.Background(), deployment)).ToNot(HaveOccurred())
deploy := &v1alpha2.WorkloadDefinition{}
deploy.Name = "deploy"
deploy.Namespace = RouteNSName
deploy.Spec.PodSpecPath = "spec.template.spec"
deploy.Spec.Reference.Name = "deployments.apps"
Expect(k8sClient.Create(context.Background(), deploy)).ToNot(HaveOccurred())
close(done)
}, 60)
var _ = AfterSuite(func() {
By("tearing down the test environment")
By("Stop the routeTrait controller")
close(controllerDone)
By("Delete the route-test namespace")
Expect(k8sClient.Delete(context.Background(), &routeNS,
client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed())
By("Tearing down the test environment")
err := testEnv.Stop()
Expect(err).ToNot(HaveOccurred())
})
pkg/controller/v1alpha1/routes/util.go
+161
View File
@@ -0,0 +1,161 @@
package routes
import (
"encoding/json"
"fmt"
"strconv"
"github.com/crossplane/oam-kubernetes-runtime/pkg/oam"
"github.com/oam-dev/kubevela/api/types"
"github.com/oam-dev/kubevela/api/v1alpha1"
"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
"github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/intstr"
)
func NeedDiscovery(routeTrait *v1alpha1.Route) bool {
if len(routeTrait.Spec.Rules) == 0 {
return true
}
for _, rule := range routeTrait.Spec.Rules {
if rule.Backend == nil {
return true
}
if rule.Backend.BackendService == nil {
return true
}
if rule.Backend.BackendService.ServiceName == "" {
return true
}
}
return false
}
func GetPodSpecPath(workloadDef *v1alpha2.WorkloadDefinition) (string, bool) {
if workloadDef.Spec.PodSpecPath != "" {
return workloadDef.Spec.PodSpecPath, true
}
if workloadDef.Labels == nil {
return "", false
}
podSpecable, ok := workloadDef.Labels[types.LabelPodSpecable]
if !ok {
return "", false
}
ok, _ = strconv.ParseBool(podSpecable)
return "", ok
}
func discoveryFromPodSpec(w *unstructured.Unstructured, fieldPath string) ([]intstr.IntOrString, error) {
paved := fieldpath.Pave(w.Object)
obj, err := paved.GetValue(fieldPath)
if err != nil {
return nil, err
}
data, err := json.Marshal(obj)
if err != nil {
return nil, fmt.Errorf("discovery podSpec from %s in workload %v err %v", fieldPath, w.GetName(), err)
}
var spec corev1.PodSpec
err = json.Unmarshal(data, &spec)
if err != nil {
return nil, fmt.Errorf("discovery podSpec from %s in workload %v err %v", fieldPath, w.GetName(), err)
}
ports := getContainerPorts(spec.Containers)
if len(ports) == 0 {
return nil, fmt.Errorf("no port found in podSpec %v", w.GetName())
}
return ports, nil
}
// discoveryFromPodTemplate not only discovery port, will also use labels in podTemplate
func discoveryFromPodTemplate(w *unstructured.Unstructured, fields ...string) ([]intstr.IntOrString, map[string]string, error) {
obj, found, _ := unstructured.NestedMap(w.Object, fields...)
if !found {
return nil, nil, fmt.Errorf("not have spec.template in workload %v", w.GetName())
}
data, err := json.Marshal(obj)
if err != nil {
return nil, nil, fmt.Errorf("workload %v convert object err %v", w.GetName(), err)
}
var spec corev1.PodTemplateSpec
err = json.Unmarshal(data, &spec)
if err != nil {
return nil, nil, fmt.Errorf("workload %v convert object to PodTemplate err %v", w.GetName(), err)
}
ports := getContainerPorts(spec.Spec.Containers)
if len(ports) == 0 {
return nil, nil, fmt.Errorf("no port found in workload %v", w.GetName())
}
return ports, spec.Labels, nil
}
func getContainerPorts(cs []corev1.Container) []intstr.IntOrString {
var ports []intstr.IntOrString
//TODO(wonderflow): exclude some sidecars
for _, container := range cs {
for _, port := range container.Ports {
ports = append(ports, intstr.FromInt(int(port.ContainerPort)))
}
}
return ports
}
// MatchService try check if the service matches the rules
func MatchService(targetPort intstr.IntOrString, rule v1alpha1.Rule) bool {
// the rule is nil, continue
if rule.Backend == nil || rule.Backend.BackendService == nil || rule.Backend.BackendService.Port.IntValue() == 0 {
return true
}
if rule.Backend.BackendService.ServiceName != "" {
return false
}
// the rule is not null, if any port matches, we regard them are all match
if targetPort == rule.Backend.BackendService.Port {
return true
}
// port is not matched, mark it not match
return false
}
func FillRouteTraitWithService(service *corev1.Service, routeTrait *v1alpha1.Route) {
if len(routeTrait.Spec.Rules) == 0 {
routeTrait.Spec.Rules = []v1alpha1.Rule{{Name: "auto-created"}}
}
for idx, rule := range routeTrait.Spec.Rules {
// If backendService.port not specified, will always use the service found and it's first port as backendService.
for _, servicePort := range service.Spec.Ports {
//We use targetPort rather than port to match with the rule, because if serviceName not specified,
//Users will only know containerPort(which is targetPort)
if MatchService(servicePort.TargetPort, rule) {
ref := &v1alpha1.BackendServiceRef{
//Use port of service rather than targetPort, it will be used in ingress pointing to the service
Port: intstr.FromInt(int(servicePort.Port)),
ServiceName: service.Name,
}
if rule.Backend == nil {
rule.Backend = &v1alpha1.Backend{BackendService: ref}
} else {
rule.Backend.BackendService = ref
}
routeTrait.Spec.Rules[idx] = rule
break
}
}
}
}
func filterLabels(labels map[string]string) map[string]string {
newLabel := make(map[string]string)
for k, v := range labels {
if k == oam.LabelOAMResourceType || k == oam.WorkloadTypeLabel {
continue
}
newLabel[k] = v
}
return newLabel
}
pkg/controller/v1alpha1/routes/util_test.go
+1
View File
@@ -0,0 +1 @@
package routes
pkg/oam/trait.go
-18
View File
@@ -2,7 +2,6 @@ package oam
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
@@ -11,8 +10,6 @@ import (
plur "github.com/gertd/go-pluralize"
"github.com/gin-gonic/gin"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/oam-dev/kubevela/api/types"
@@ -21,21 +18,6 @@ import (
"github.com/oam-dev/kubevela/pkg/server/apis"
)
func GetTraitDefNameFromRaw(extension runtime.RawExtension) string {
if extension.Raw == nil {
extension.Raw, _ = extension.MarshalJSON()
}
var data map[string]interface{}
// leverage Admission Controller to do the check
_ = json.Unmarshal(extension.Raw, &data)
obj := unstructured.Unstructured{Object: data}
ann := obj.GetAnnotations()
if ann == nil {
return obj.GetKind()
}
return ann[types.AnnTraitDef]
}
func ListTraitDefinitions(workloadName *string) ([]types.Capability, error) {
var traitList []types.Capability
traits, err := plugins.LoadInstalledCapabilityWithType(types.TypeTrait)