From a48b74d4bebbdf6244052f6b4848b5a78fc344ce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 9 Oct 2021 15:24:41 +0800 Subject: [PATCH] Feat: add secure tls for cluster-gateway (#2428) (cherry picked from commit cf4d508189173537b3707c62c88fb981437e42b9) Co-authored-by: Yin Da --- .../vela-core/templates/cluster-gateway.yaml | 105 +++++++++++++++- charts/vela-core/values.yaml | 7 +- .../templates/cluster-gateway.yaml | 115 +++++++++++++++++- charts/vela-minimal/values.yaml | 7 +- 4 files changed, 223 insertions(+), 11 deletions(-) diff --git a/charts/vela-core/templates/cluster-gateway.yaml b/charts/vela-core/templates/cluster-gateway.yaml index 944c59052..9590b5a45 100644 --- a/charts/vela-core/templates/cluster-gateway.yaml +++ b/charts/vela-core/templates/cluster-gateway.yaml @@ -32,12 +32,28 @@ spec: - "--secure-port={{ .Values.multicluster.clusterGateway.port }}" - "--secret-namespace={{ .Release.Namespace }}" - "--feature-gates=APIPriorityAndFairness=false" + {{ if .Values.multicluster.clusterGateway.secureTLS.enabled }} + - "--cert-dir={{ .Values.multicluster.clusterGateway.secureTLS.certPath }}" + {{ end }} image: {{ .Values.multicluster.clusterGateway.image.repository }}:{{ .Values.multicluster.clusterGateway.image.tag }} imagePullPolicy: {{ .Values.multicluster.clusterGateway.image.pullPolicy }} resources: {{- toYaml .Values.multicluster.clusterGateway.resources | nindent 12 }} ports: - containerPort: {{ .Values.multicluster.clusterGateway.port }} + {{ if .Values.multicluster.clusterGateway.secureTLS.enabled }} + volumeMounts: + - mountPath: {{ .Values.multicluster.clusterGateway.secureTLS.certPath }} + name: tls-cert-vol + readOnly: true + {{- end }} + {{ if .Values.multicluster.clusterGateway.secureTLS.enabled }} + volumes: + - name: tls-cert-vol + secret: + defaultMode: 420 + secretName: {{ template "kubevela.fullname" . }}-cluster-gateway-tls + {{ end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -84,5 +100,92 @@ spec: namespace: {{ .Release.Namespace }} port: {{ .Values.multicluster.clusterGateway.port }} versionPriority: 10 - insecureSkipTLSVerify: true + insecureSkipTLSVerify: {{ not .Values.multicluster.clusterGateway.secureTLS.enabled }} + {{ if .Values.multicluster.clusterGateway.secureTLS.enabled }} + caBundle: Cg== + {{ end }} {{ end }} +--- +{{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-create + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-create + {{- include "kubevela.labels" . | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-create + labels: + app: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-create + {{- include "kubevela.labels" . | nindent 8 }} + spec: + containers: + - name: create + image: {{ .Values.admissionWebhooks.patch.image.repository }}:{{ .Values.admissionWebhooks.patch.image.tag }} + imagePullPolicy: {{ .Values.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{ .Release.Name }}-cluster-gateway-service,{{ .Release.Name }}-cluster-gateway-service.{{ .Release.Namespace }}.svc + - --namespace={{ .Release.Namespace }} + - --secret-name={{ template "kubevela.fullname" . }}-cluster-gateway-tls + - --key-name=apiserver.key + - --cert-name=apiserver.crt + restartPolicy: OnFailure + serviceAccountName: {{ template "kubevela.fullname" . }}-admission + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 +{{ end }} +--- +{{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-patch + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-patch + {{- include "kubevela.labels" . | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-patch + labels: + app: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-patch + {{- include "kubevela.labels" . | nindent 8 }} + spec: + containers: + - name: patch + image: {{ .Values.multicluster.clusterGateway.image.repository }}:{{ .Values.multicluster.clusterGateway.image.tag }} + imagePullPolicy: {{ .Values.multicluster.clusterGateway.image.pullPolicy }} + command: + - /patch + args: + - --secret-namespace={{ .Release.Namespace }} + - --secret-name={{ template "kubevela.fullname" . }}-cluster-gateway-tls + restartPolicy: OnFailure + serviceAccountName: {{ include "kubevela.serviceAccountName" . }} + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 +{{ end }} \ No newline at end of file diff --git a/charts/vela-core/values.yaml b/charts/vela-core/values.yaml index eae8f91e7..10d6800e9 100644 --- a/charts/vela-core/values.yaml +++ b/charts/vela-core/values.yaml @@ -118,9 +118,12 @@ multicluster: port: 9443 image: repository: oamdev/cluster-gateway - tag: v1.1.2 + tag: v1.1.3 pullPolicy: Always resources: limits: cpu: 100m - memory: 200Mi \ No newline at end of file + memory: 200Mi + secureTLS: + enabled: true + certPath: /etc/k8s-cluster-gateway-certs \ No newline at end of file diff --git a/charts/vela-minimal/templates/cluster-gateway.yaml b/charts/vela-minimal/templates/cluster-gateway.yaml index 944c59052..817036913 100644 --- a/charts/vela-minimal/templates/cluster-gateway.yaml +++ b/charts/vela-minimal/templates/cluster-gateway.yaml @@ -32,12 +32,28 @@ spec: - "--secure-port={{ .Values.multicluster.clusterGateway.port }}" - "--secret-namespace={{ .Release.Namespace }}" - "--feature-gates=APIPriorityAndFairness=false" + {{ if .Values.multicluster.clusterGateway.secureTLS.enabled }} + - "--cert-dir={{ .Values.multicluster.clusterGateway.secureTLS.certPath }}" + {{ end }} image: {{ .Values.multicluster.clusterGateway.image.repository }}:{{ .Values.multicluster.clusterGateway.image.tag }} imagePullPolicy: {{ .Values.multicluster.clusterGateway.image.pullPolicy }} resources: {{- toYaml .Values.multicluster.clusterGateway.resources | nindent 12 }} ports: - containerPort: {{ .Values.multicluster.clusterGateway.port }} + {{ if .Values.multicluster.clusterGateway.secureTLS.enabled }} + volumeMounts: + - mountPath: {{ .Values.multicluster.clusterGateway.secureTLS.certPath }} + name: tls-cert-vol + readOnly: true + {{- end }} + {{ if .Values.multicluster.clusterGateway.secureTLS.enabled }} + volumes: + - name: tls-cert-vol + secret: + defaultMode: 420 + secretName: {{ template "kubevela.fullname" . }}-cluster-gateway-tls + {{ end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -50,9 +66,9 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} -{{ end }} + {{ end }} --- -{{ if .Values.multicluster.enabled }} + {{ if .Values.multicluster.enabled }} apiVersion: v1 kind: Service metadata: @@ -65,9 +81,9 @@ spec: - protocol: TCP port: {{ .Values.multicluster.clusterGateway.port }} targetPort: {{ .Values.multicluster.clusterGateway.port }} -{{ end }} + {{ end }} --- -{{ if .Values.multicluster.enabled }} + {{ if .Values.multicluster.enabled }} apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: @@ -84,5 +100,92 @@ spec: namespace: {{ .Release.Namespace }} port: {{ .Values.multicluster.clusterGateway.port }} versionPriority: 10 - insecureSkipTLSVerify: true -{{ end }} + insecureSkipTLSVerify: {{ not .Values.multicluster.clusterGateway.secureTLS.enabled }} + {{ if .Values.multicluster.clusterGateway.secureTLS.enabled }} + caBundle: Cg== + {{ end }} + {{ end }} +--- + {{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-create + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-create + {{- include "kubevela.labels" . | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-create + labels: + app: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-create + {{- include "kubevela.labels" . | nindent 8 }} + spec: + containers: + - name: create + image: {{ .Values.admissionWebhooks.patch.image.repository }}:{{ .Values.admissionWebhooks.patch.image.tag }} + imagePullPolicy: {{ .Values.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{ .Release.Name }}-cluster-gateway-service,{{ .Release.Name }}-cluster-gateway-service.{{ .Release.Namespace }}.svc + - --namespace={{ .Release.Namespace }} + - --secret-name={{ template "kubevela.fullname" . }}-cluster-gateway-tls + - --key-name=apiserver.key + - --cert-name=apiserver.crt + restartPolicy: OnFailure + serviceAccountName: {{ template "kubevela.fullname" . }}-admission + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + {{ end }} +--- + {{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-patch + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-patch + {{- include "kubevela.labels" . | nindent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-patch + labels: + app: {{ template "kubevela.fullname" . }}-cluster-gateway-tls-secret-patch + {{- include "kubevela.labels" . | nindent 8 }} + spec: + containers: + - name: patch + image: {{ .Values.multicluster.clusterGateway.image.repository }}:{{ .Values.multicluster.clusterGateway.image.tag }} + imagePullPolicy: {{ .Values.multicluster.clusterGateway.image.pullPolicy }} + command: + - /patch + args: + - --secret-namespace={{ .Release.Namespace }} + - --secret-name={{ template "kubevela.fullname" . }}-cluster-gateway-tls + restartPolicy: OnFailure + serviceAccountName: {{ include "kubevela.serviceAccountName" . }} + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + {{ end }} \ No newline at end of file diff --git a/charts/vela-minimal/values.yaml b/charts/vela-minimal/values.yaml index 407bcf374..104a3b7c1 100644 --- a/charts/vela-minimal/values.yaml +++ b/charts/vela-minimal/values.yaml @@ -117,9 +117,12 @@ multicluster: port: 9443 image: repository: oamdev/cluster-gateway - tag: v1.1.2 + tag: v1.1.3 pullPolicy: Always resources: limits: cpu: 100m - memory: 200Mi \ No newline at end of file + memory: 200Mi + secureTLS: + enabled: true + certPath: /etc/k8s-cluster-gateway-certs \ No newline at end of file