From 2beef8b2aa8fc2a84e6a106f01cab027dc3a9857 Mon Sep 17 00:00:00 2001 From: Thorsten Kunz <30314770+ThorstenKunz@users.noreply.github.com> Date: Fri, 26 Oct 2018 18:50:31 +0200 Subject: [PATCH] [stable/fluent-bit] Fix ElasticSearch Output config templatate (#8166) The most important part of this PR is the fix for the ES config which was broken because it was cut off by the Splunk config. It also adds the tls options to the http and fluentd output plugin as well as a shared_secret option to the fluentd forwarder output plugin. Also http_user/password options are added to the http backend. All options have backwards compatible defauts. Signed-off-by: Thorsten Kunz --- stable/fluent-bit/Chart.yaml | 2 +- stable/fluent-bit/README.md | 10 +++++++ stable/fluent-bit/templates/config.yaml | 37 +++++++++++++++---------- stable/fluent-bit/values.yaml | 12 +++++++- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/stable/fluent-bit/Chart.yaml b/stable/fluent-bit/Chart.yaml index d4810a0df8..a46bab3bef 100755 --- a/stable/fluent-bit/Chart.yaml +++ b/stable/fluent-bit/Chart.yaml @@ -1,5 +1,5 @@ name: fluent-bit -version: 0.13.1 +version: 0.14.0 appVersion: 0.14.4 description: Fast and Lightweight Log/Data Forwarder for Linux, BSD and OSX keywords: diff --git a/stable/fluent-bit/README.md b/stable/fluent-bit/README.md index d29b751fa9..ac2eabbd25 100644 --- a/stable/fluent-bit/README.md +++ b/stable/fluent-bit/README.md @@ -34,6 +34,10 @@ The following table lists the configurable parameters of the Fluent-Bit chart an | **Forward Backend** | | `backend.forward.host` | Target host where Fluent-Bit or Fluentd are listening for Forward messages | `fluentd` | | `backend.forward.port` | TCP Port of the target service | `24284` | +| `backend.forward.shared_key` | A key string known by the remote Fluentd used for authorization. | `` | +| `backend.forward.tls` | Enable or disable TLS support | `off` | +| `backend.forward.tls_verify` | Force certificate validation | `on` | +| `backend.forward.tls_debug` | Set TLS debug verbosity level. It accept the following values: 0-4 | `1` | | **ElasticSearch Backend** | | `backend.es.host` | IP address or hostname of the target Elasticsearch instance | `elasticsearch` | | `backend.es.port` | TCP port of the target Elasticsearch instance. | `9200` | @@ -51,7 +55,12 @@ The following table lists the configurable parameters of the Fluent-Bit chart an | `backend.http.host` | IP address or hostname of the target HTTP Server | `127.0.0.1` | | `backend.http.port` | TCP port of the target HTTP Server | `80` | | `backend.http.uri` | Specify an optional HTTP URI for the target web server, e.g: /something | `"/"` +| `backend.http.http_user` | Optional username credential for Basic Authentication. | `` | +| `backend.http.http_passwd:` | Password for user defined in HTTP_User. | `` | | `backend.http.format` | Specify the data format to be used in the HTTP request body, by default it uses msgpack, optionally it can be set to json. | `msgpack` | +| `backend.http.tls` | Enable or disable TLS support | `off` | +| `backend.http.tls_verify` | Force certificate validation | `on` | +| `backend.http.tls_debug` | Set TLS debug verbosity level. It accept the following values: 0-4 | `1` | | **Splunk Backend** | | `backend.splunk.host` | IP address or hostname of the target Splunk Server | `127.0.0.1` | | `backend.splunk.port` | TCP port of the target Splunk Server | `8088` | @@ -59,6 +68,7 @@ The following table lists the configurable parameters of the Fluent-Bit chart an | `backend.splunk.send_raw` | If enabled, record keys and values are set in the main map. | `off` | | `backend.splunk.tls` | Enable or disable TLS support | `on` | | `backend.splunk.tls_verify` | Force TLS certificate validation | `off` | +| `backend.splunk.tls_debug` | Set TLS debug verbosity level. It accept the following values: 0-4 | `1` | | `backend.splunk.message_key` | Tag applied to all incoming logs | `kubernetes` | | **Parsers** | | `parsers.enabled` | Enable custom parsers | `false` | diff --git a/stable/fluent-bit/templates/config.yaml b/stable/fluent-bit/templates/config.yaml index e25fb6c09f..3bd926e1c4 100644 --- a/stable/fluent-bit/templates/config.yaml +++ b/stable/fluent-bit/templates/config.yaml @@ -45,7 +45,7 @@ data: Kube_CA_File {{ .Values.filter.kubeCAFile }} Kube_Token_File {{ .Values.filter.kubeTokenFile }} {{- if .Values.filter.mergeJSONLog }} - Merge_JSON_Log On + Merge_Log On {{- end }} {{- if .Values.filter.enableParser }} K8S-Logging.Parser On @@ -67,6 +67,9 @@ data: Host {{ .Values.backend.forward.host }} Port {{ .Values.backend.forward.port }} Retry_Limit False +{{- if .Values.backend.forward.shared_key }} + Shared_Key {{ .Values.backend.forward.shared_key }} +{{- end }} {{ else if eq .Values.backend.type "es" }} [OUTPUT] Name es @@ -79,18 +82,6 @@ data: {{- if .Values.backend.es.time_key }} Time_Key {{ .Values.backend.es.time_key }} {{- end }} -{{ else if eq .Values.backend.type "splunk" }} - [OUTPUT] - Name splunk - Match * - Host {{ .Values.backend.splunk.host }} - Port {{ .Values.backend.splunk.port }} - Splunk_Token {{ .Values.backend.splunk.token }} - Splunk_Send_Raw {{ .Values.backend.splunk.send_raw}} - TLS {{ .Values.backend.splunk.tls }} - TLS.Verify {{ .Values.backend.splunk.tls_verify }} - Message_Key {{ .Values.backend.splunk.message_key }} - {{- if .Values.backend.es.logstash_prefix }} Logstash_Prefix {{ .Values.backend.es.logstash_prefix }} {{ else if .Values.backend.es.index }} @@ -108,7 +99,18 @@ data: tls.ca_file /secure/es-tls-ca.crt {{- end }} {{- end }} - +{{ else if eq .Values.backend.type "splunk" }} + [OUTPUT] + Name splunk + Match * + Host {{ .Values.backend.splunk.host }} + Port {{ .Values.backend.splunk.port }} + Splunk_Token {{ .Values.backend.splunk.token }} + Splunk_Send_Raw {{ .Values.backend.splunk.send_raw}} + TLS {{ .Values.backend.splunk.tls }} + TLS.Verify {{ .Values.backend.splunk.tls_verify }} + tls.debug {{ .Values.backend.splunk.tls_debug }} + Message_Key {{ .Values.backend.splunk.message_key }} {{ else if eq .Values.backend.type "http" }} [OUTPUT] Name http @@ -116,6 +118,13 @@ data: Host {{ .Values.backend.http.host }} Port {{ .Values.backend.http.port }} URI {{ .Values.backend.http.uri }} +{{- if .Values.backend.http.http_user }} + HTTP_User {{ .Values.backend.http.http_user }} + HTTP_Passwd {{ .Values.backend.http.http_passwd }} +{{- end }} + tls {{ .Values.backend.http.tls }} + tls.verify {{ .Values.backend.http.tls_verify }} + tls.debug {{ .Values.backend.http.tls_debug }} {{- if .Values.backend.http.proxy }} Proxy {{ .Values.backend.http.proxy }} {{- end }} diff --git a/stable/fluent-bit/values.yaml b/stable/fluent-bit/values.yaml index 07df5eae70..c7954c66e2 100644 --- a/stable/fluent-bit/values.yaml +++ b/stable/fluent-bit/values.yaml @@ -23,6 +23,10 @@ backend: forward: host: fluentd port: 24284 + tls: "off" + tls_verify: "on" + tls_debug: 1 + shared_key: es: host: elasticsearch port: 9200 @@ -40,7 +44,7 @@ backend: tls_verify: "on" # TLS certificate for the Elastic (in PEM format). Use if tls=on and tls_verify=on. tls_ca: "" - # TLS debugging levels = 1-5 + # TLS debugging levels = 1-4 tls_debug: 1 splunk: host: 127.0.0.1 @@ -49,6 +53,7 @@ backend: send_raw: "on" tls: "on" tls_verify: "off" + tls_debug: 1 message_key: "kubernetes" ## @@ -58,6 +63,11 @@ backend: host: 127.0.0.1 port: 80 uri: "/" + http_user: + http_passwd: + tls: "off" + tls_verify: "on" + tls_debug: 1 ## Specify the data format to be used in the HTTP request body ## Can be either 'msgpack' or 'json' format: msgpack