From 7528f2ec6dc764a3ea1420fed61dd904143b7177 Mon Sep 17 00:00:00 2001 From: "D. Lee" Date: Fri, 10 Mar 2023 14:34:37 -0800 Subject: [PATCH] Add lightweight logging mode (#668) Enabled with `logging.fileMode`: true - disables elasticsearch, kibana and ingress - only enables fluentd to write logs in the node's volume - lightweight logging into files (in JSON format and compressed in gzip) - log file rotation (default: rotating files every 4 hours, retention 3 days) --- chart/admin/logging/README.md | 7 +++ chart/admin/logging/templates/fluentd.yaml | 71 ++++++++++++++++++++-- chart/admin/logging/templates/logging.yaml | 3 + chart/admin/logging/values.yaml | 7 ++- 4 files changed, 79 insertions(+), 9 deletions(-) diff --git a/chart/admin/logging/README.md b/chart/admin/logging/README.md index ad9fcbc1..6fd3e31d 100644 --- a/chart/admin/logging/README.md +++ b/chart/admin/logging/README.md @@ -20,6 +20,13 @@ And, edit `chart/examples/local-logging.yaml` for a local test. Optionally, when install the logging service only, edit `chart/admin/logging/values.yaml`. For a local test, it should use a hostname (not `localhost` but a hostname like `myhostname` registered in `/etc/hosts`) +# Modes + +* Lightweight File mode (Fluentd only mode): set `logging.fileMode` to `true` + * This will disable Elasticsearch, Kibana and Ingress. + * Log files will be placed in each node's `/var/log/fluentd/`. + * Log file's retention period: 3 days (see `templates/fluentd.yaml`) + ## Installation * run a setup script (will create a namespace and install elastic's CRDS) diff --git a/chart/admin/logging/templates/fluentd.yaml b/chart/admin/logging/templates/fluentd.yaml index 7acdf27a..1f48a03e 100644 --- a/chart/admin/logging/templates/fluentd.yaml +++ b/chart/admin/logging/templates/fluentd.yaml @@ -35,6 +35,52 @@ subjects: - kind: ServiceAccount name: fluentd namespace: {{ .Values.logging.namespace | default "btrix-admin" }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-config + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} + labels: + k8s-app: fluentd-logging +data: + fluent.conf: | + @include "#{ENV['FLUENTD_SYSTEMD_CONF'] || 'systemd'}.conf" + @include "#{ENV['FLUENTD_PROMETHEUS_CONF'] || 'prometheus'}.conf" + @include kubernetes.conf + @include conf.d/*.conf + + + @type file + path /var/log/fluentd/btrix.log + time_slice_format %Y%m%d%H%M%S + time_slice_wait 10m + compress gzip + + + @type json + + + + @type file + path /var/log/fluentd/buffer + flush_thread_count 2 + flush_interval 5s + chunk_limit_size 1m + queue_limit_length 64 + overflow_action block + + + + @type time + timekey %Y%m%d%H%M + # How often rotate a file. 14400 means 4 hour. default is 86400 seconds (1 day) + interval 14400 + # How long to keep rotated files. 0 means forever. default is 7 days + keep 3 + + --- apiVersion: apps/v1 kind: DaemonSet @@ -74,6 +120,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + {{ if not .Values.logging.fileMode }} - name: FLUENT_ELASTICSEARCH_HOST value: "btrixlog-es-http.{{ .Values.logging.namespace | default "btrix-admin" }}.svc.cluster.local" - name: FLUENT_ELASTICSEARCH_PORT @@ -99,6 +146,7 @@ spec: secretKeyRef: name: btrixlog-es-elastic-user key: elastic + {{ end }} # ===================== - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH value: /var/log/containers/fluent* @@ -114,6 +162,12 @@ spec: cpu: {{ .Values.logging.fluentd.cpu | default "60m" }} memory: {{ .Values.logging.fluentd.mem | default "200Mi" }} volumeMounts: + {{ if .Values.logging.fileMode }} + - name: config-volume + mountPath: /fluentd/etc/fluent.conf + subPath: fluent.conf + readOnly: true + {{ end }} - name: varlog mountPath: {{ .Values.logging.fluentd.logVar | default "/var/log" }} # When actual pod logs in /var/lib/docker/containers, the following lines should be used. @@ -121,11 +175,16 @@ spec: mountPath: {{ .Values.logging.fluentd.logPathContainers | default "/var/lib/docker/containers" }} readOnly: true # When actual pod logs in /var/log/pods, the following lines should be used. - # - name: dockercontainerlogdirectory - # mountPath: /var/log/pods - # readOnly: true + - name: dockercontainerlogdirectory2 + mountPath: /var/log/pods + readOnly: true terminationGracePeriodSeconds: 30 volumes: + {{ if .Values.logging.fileMode }} + - name: config-volume + configMap: + name: fluentd-config + {{ end }} - name: varlog hostPath: path: {{ .Values.logging.fluentd.logVar | default "/var/log" }} @@ -134,7 +193,7 @@ spec: hostPath: path: {{ .Values.logging.fluentd.logPathContainers | default "/var/lib/docker/containers" }} # When actual pod logs in /var/log/pods, the following lines should be used. - # - name: dockercontainerlogdirectory - # hostPath: - # path: /var/log/pods + - name: dockercontainerlogdirectory2 + hostPath: + path: /var/log/pods {{- end -}} \ No newline at end of file diff --git a/chart/admin/logging/templates/logging.yaml b/chart/admin/logging/templates/logging.yaml index cbc95668..7fae02b2 100644 --- a/chart/admin/logging/templates/logging.yaml +++ b/chart/admin/logging/templates/logging.yaml @@ -1,8 +1,11 @@ {{ if .Values.logging.enabled }} +{{ if not .Values.logging.fileMode }} {{ include "es.install" . }} {{ include "kb.install" . }} {{ include "ingress.install" . }} +{{ end }} + {{ include "fluentd.install" . }} {{ end }} diff --git a/chart/admin/logging/values.yaml b/chart/admin/logging/values.yaml index d8b43fcd..9b4581d3 100644 --- a/chart/admin/logging/values.yaml +++ b/chart/admin/logging/values.yaml @@ -2,21 +2,22 @@ logging: namespace: btrix-admin enabled: true dedicatedNode: - enabled: false + enabled: true nodeType: admin + fileMode: true ingress: tls: false host: localhost path: /kibana elasticsearch: - local: true + local: false cpu: 1 mem: 4Gi opt: -Xms2g -Xmx2g volumeEnabled: false volumeSize: 1Gi kibana: - local: true + local: false cpu: 1 mem: 1Gi opt: --max-old-space-size=1024