add logging

2022-12-29 09:22:04 -08:00 · 2022-12-29 09:22:04 -08:00 · 5b7d214c8a
commit 5b7d214c8a
parent 4a23dd12cb
20 changed files with 524 additions and 0 deletions
--- a/chart/Chart.lock
+++ b/chart/Chart.lock
@ -0,0 +1,6 @@
+dependencies:
+- name: btrix-admin-logging
+  repository: file://./admin/logging
+  version: 0.1.0
+digest: sha256:a62579385835818c52ddc8c59d6ec5a17800e60b71a2e75b3a85a02c262bb385
+generated: "2023-01-03T00:22:20.096291-08:00"
--- a/chart/Chart.yaml
+++ b/chart/Chart.yaml
@ -14,3 +14,8 @@ version: 0.1.0
 # follow Semantic Versioning. They should reflect the version the application is using.
 appVersion: 0.1.0

+dependencies:
+  - name: btrix-admin-logging
+    version: 0.1.0
+    condition: addons.admin.logging
+    repository: file://./admin/logging
--- a/chart/README.md
+++ b/chart/README.md
@ -0,0 +1,7 @@
+## Update Helm dependencies
+
+* It needs to update Helm charts after changing its dependencies (e.g. logging)
+
+```
+$ helm dependency update .
+```
--- a/chart/admin/logging/.gitignore
+++ b/chart/admin/logging/.gitignore
@ -0,0 +1,2 @@
+old
+_tmp_prod_
--- a/chart/admin/logging/Chart.yaml
+++ b/chart/admin/logging/Chart.yaml
@ -0,0 +1,15 @@
+apiVersion: v2
+name: btrix-admin-logging
+description: A chart for running the Webrecorder Browsertrix System - admin services
+type: application
+icon: https://webrecorder.net/assets/icon.png
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 0.1.0
--- a/chart/admin/logging/README.md
+++ b/chart/admin/logging/README.md
@ -0,0 +1,74 @@
+# Btrix Logging Service
+
+## Prerequisites
+
+1. [optional] add a new node for a dedicated logging stacks (Elasticsearch, Fluentd, Kibana)
+1. [optional] set a label for a node with `NodeType=admin` (when `dedicatedNode` is `true`)
+```
+kubectl label nodes new-admin-node nodeType=admin
+```
+1. edit `chart/values.yaml` to enable `logging` service when install with browsestrix-cloud
+```
+addons:
+  admin:
+    logging: true
+```
+This will enable the helm dependencies defined in `chart/Chart.yaml`.
+
+And, edit `chart/examples/local-logging.yaml` for a local test.
+
+Optionally, when install the logging service only, edit `chart/admin/logging/values.yaml`.
+For a local test, it should use a hostname (not `localhost` but a hostname registered in `/etc/hosts`)
+
+## Installation
+
+* run a setup script (will create a namespace and install elastic's CRDS)
+```
+$ ./chart/admin/logging/scripts/eck_install.sh
+```
+* install logging services using helm chart
+```
+helm upgrade --install -f ./chart/values.yaml -f ./chart/examples/local-logging.yaml  btrix ./chart
+```
+
+## Installation (logging service only)
+
+* run a setup script (will create a namespace and install elastic's CRDS)
+```
+$ ./chart/admin/logging/scripts/eck_install.sh
+```
+* install logging services using helm chart
+```
+helm upgrade --install -f ./chart/admin/logging/values.yaml btrix-admin-log ./chart/admin/logging
+```
+
+## Access Kibana dashboard
+
+* get the Kibana's login password (username: `elastic`)
+```
+kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}'
+```
+* open `https://hostname/kibana`
+
+## Import/Export Kibana data
+
+* Import data (e.g. data view, search queries and dashboards)
+
+```
+cd ./chart/admin/logging/scripts
+./kibana_imports.sh
+```
+
+* Exports data (e.g. data view, search queries and dashboards)
+
+```
+cd ./chart/admin/logging/scripts
+./kibana_exports.sh
+```
+
+## Uninstallation
+
+```
+$ helm uninstall btrix-admin-log
+$ ./chart/admin/logging/scripts/eck_uninstall.sh
+```
--- a/chart/admin/logging/scripts/eck_install.sh
+++ b/chart/admin/logging/scripts/eck_install.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+kubectl create namespace btrix-admin
+kubectl create -f https://download.elastic.co/downloads/eck/2.5.0/crds.yaml
+kubectl apply -f https://download.elastic.co/downloads/eck/2.5.0/operator.yaml
+
+kubectl label nodes docker-desktop nodeType=admin
+kubectl get nodes -o wide -o jsonpath='{.items[*].metadata.labels}' | jq .
--- a/chart/admin/logging/scripts/eck_uninstall.sh
+++ b/chart/admin/logging/scripts/eck_uninstall.sh
@ -0,0 +1,5 @@
+#!/bin/bash
+
+kubectl delete -f https://download.elastic.co/downloads/eck/2.5.0/operator.yaml
+kubectl delete -f https://download.elastic.co/downloads/eck/2.5.0/crds.yaml
+kubectl delete namespace btrix-admin
--- a/chart/admin/logging/scripts/kibana_export.ndjson
+++ b/chart/admin/logging/scripts/kibana_export.ndjson
@ -0,0 +1,8 @@
+{"attributes":{"fieldAttrs":"{\"kubernetes.host\":{\"count\":2},\"kubernetes.labels.app\":{\"count\":2},\"kubernetes.pod_name\":{\"count\":3},\"log\":{\"count\":5},\"tag\":{\"count\":2}}","fieldFormatMap":"{}","fields":"[]","name":"btrix-cloud","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"@timestamp","title":"logstash-*","typeMeta":"{}"},"coreMigrationVersion":"8.5.3","id":"4af04743-fb92-454c-9e13-75dc6422e01b","migrationVersion":{"index-pattern":"8.0.0"},"references":[],"type":"index-pattern","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2NSwxXQ=="}
+{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"8.5.3\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":15,\"i\":\"7447c31f-5270-437a-a849-aaad2c524bc3\"},\"panelIndex\":\"7447c31f-5270-437a-a849-aaad2c524bc3\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"4af04743-fb92-454c-9e13-75dc6422e01b\",\"name\":\"indexpattern-datasource-layer-f89b17c5-99c7-4348-972f-b072475a449f\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"f89b17c5-99c7-4348-972f-b072475a449f\",\"accessors\":[\"0a3522e2-426b-4246-a506-b8dddf6ae0e2\"],\"position\":\"top\",\"seriesType\":\"line\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"293a66d9-5cb2-46f2-b204-99b2d91f693b\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"f89b17c5-99c7-4348-972f-b072475a449f\":{\"columns\":{\"293a66d9-5cb2-46f2-b204-99b2d91f693b\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"0a3522e2-426b-4246-a506-b8dddf6ae0e2\":{\"label\":\"Count of records\",\"customLabel\":false,\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"293a66d9-5cb2-46f2-b204-99b2d91f693b\",\"0a3522e2-426b-4246-a506-b8dddf6ae0e2\"],\"incompleteColumns\":{}}}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Graph\"}]","timeRestore":false,"title":"btrix-cloud","version":1},"coreMigrationVersion":"8.5.3","id":"b2460db0-8629-11ed-a271-e3c0b9803788","migrationVersion":{"dashboard":"8.5.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"7447c31f-5270-437a-a849-aaad2c524bc3:indexpattern-datasource-layer-f89b17c5-99c7-4348-972f-b072475a449f","type":"index-pattern"}],"type":"dashboard","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2NiwxXQ=="}
+{"attributes":{"columns":["log","kubernetes.pod_name"],"description":"","grid":{"columns":{"log":{"width":1061}}},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.namespace_name.keyword: crawlers AND ERROR\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-crawlers-errors","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"c08d4250-861d-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2NywxXQ=="}
+{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.pod_name.keyword: job-*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-crawl-jobs","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"c132b4a0-861e-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2OCwxXQ=="}
+{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.namespace_name.keyword: crawlers\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-crawlers","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"645f4720-861e-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2OSwxXQ=="}
+{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.pod_name.keyword : browsertrix-cloud-frontend-*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-frontend","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"04c7feb0-861e-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk3MCwxXQ=="}
+{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.pod_name.keyword : browsertrix-cloud-backend-*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-backend","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"f3cf5c20-861d-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk3MSwxXQ=="}
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":7,"missingRefCount":0,"missingReferences":[]}
--- a/chart/admin/logging/scripts/kibana_exports.sh
+++ b/chart/admin/logging/scripts/kibana_exports.sh
@ -0,0 +1,20 @@
+#!/bin/bash
+
+ES_USER="elastic"
+ES_PASS=$(kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}')
+KIBANA_INGRESS="kibana-main"
+HOSTNAME=`kubectl get ingress -A | grep $KIBANA_INGRESS | awk '{print $4}'`
+KIBANA_URL="https://${HOSTNAME}/kibana"
+EXPORT_FN="./kibana_export.ndjson"
+
+echo "use $KIBANA_URL"
+
+RET=`curl --user $ES_USER:$ES_PASS \
+    "${KIBANA_URL}/api/kibana/management/saved_objects/_find?perPage=50&page=1&fields=id&type=url&type=index-pattern&type=action&type=query&type=alert&type=search&type=graph-workspace&type=tag&type=csp_rule&type=csp-rule-template&type=visualization&type=canvas-element&type=canvas-workpad&type=dashboard&type=lens&type=map&type=cases&type=osquery-saved-query&type=osquery-pack&type=uptime-dynamic-settings&type=synthetics-privates-locations&type=infrastructure-ui-source&type=metrics-explorer-view&type=inventory-view&type=infrastructure-monitoring-log-view&type=apm-indices&sortField=updated_at&sortOrder=desc" | \
+    jq -r ".saved_objects | [.[] | { id:.id, type:.type}] | @json"`
+
+curl --user $ES_USER:$ES_PASS -X POST \
+    "${KIBANA_URL}/api/saved_objects/_export" \
+    -H 'kbn-xsrf: true' \
+    -H 'Content-Type: application/json' \
+    -d "{\"objects\": $RET }" > $EXPORT_FN
--- a/chart/admin/logging/scripts/kibana_imports.sh
+++ b/chart/admin/logging/scripts/kibana_imports.sh
@ -0,0 +1,15 @@
+#!/bin/bash
+
+ES_USER="elastic"
+ES_PASS=$(kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}')
+KIBANA_INGRESS="kibana-main"
+HOSTNAME=`kubectl get ingress -A | grep $KIBANA_INGRESS | awk '{print $4}'`
+KIBANA_URL="https://${HOSTNAME}/kibana"
+EXPORT_FN="./kibana_export.ndjson"
+
+echo "use $KIBANA_URL"
+
+curl -k --user $ES_USER:$ES_PASS -X POST \
+    "${KIBANA_URL}/api/saved_objects/_import" \
+    -H "kbn-xsrf: true" \
+    --form file=@$EXPORT_FN
--- a/chart/admin/logging/templates/elasticsearch.yaml
+++ b/chart/admin/logging/templates/elasticsearch.yaml
@ -0,0 +1,70 @@
+{{- define "es.install" -}}
+{{ if .Values.logging.elasticsearch.volumeEnabled }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: es-storage-pvc
+  namespace: {{ .Values.logging.namespace | default "btrix-admin" }}
+  annotations:
+    "helm.sh/resource-policy": keep
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.logging.elasticsearch.volumeSize | default "1Gi" }}
+  storageClassName: hostpath
+{{ end }}
+---
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: btrixlog
+  namespace: {{ .Values.logging.namespace | default "btrix-admin" }}
+spec:
+  version: 8.5.3
+  nodeSets:
+  - name: default
+    count: 1
+    config:
+      node.store.allow_mmap: false
+    podTemplate:
+      spec:
+        {{ if .Values.logging.elasticsearch.volumeEnabled }}
+        volumes:
+          - name: data-es
+            persistentVolumeClaim:
+              claimName: es-storage-pvc
+        {{ end }}
+        {{ if .Values.logging.dedicatedNode.enabled }}
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: nodeType
+                  operator: In
+                  values:
+                  - "{{ .Values.logging.dedicatedNode.nodeType | default "admin" }}"
+        {{ end }}
+        containers:
+        - name: elasticsearch
+          {{ if not .Values.logging.elasticsearch.local }}
+          env:
+          - name: ES_JAVA_OPTS
+            value: "{{ .Values.logging.elasticsearch.opt | default "-Xms2g -Xmx2g" }}"
+          resources:
+            requests:
+              memory: {{ .Values.logging.elasticsearch.mem | default "4Gi" }}
+              cpu: {{ .Values.logging.elasticsearch.cpu | default "1" }}
+            limits:
+              memory: {{ .Values.logging.elasticsearch.mem | default "4Gi" }}
+              cpu: {{ .Values.logging.elasticsearch.cpu | default "1" }}
+          {{ end }}
+          {{ if .Values.logging.elasticsearch.volumeEnabled }}
+          volumeMounts:
+            - name: data-es
+              mountPath: /usr/share/elasticsearch/data
+          {{ end }}
+{{- end -}}
--- a/chart/admin/logging/templates/fluentd.yaml
+++ b/chart/admin/logging/templates/fluentd.yaml
@ -0,0 +1,138 @@
+{{- define "fluentd.install" -}}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluentd
+  namespace: {{ .Values.logging.namespace | default "btrix-admin" }}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: fluentd
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: fluentd
+roleRef:
+  kind: ClusterRole
+  name: fluentd
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: fluentd
+  namespace: {{ .Values.logging.namespace | default "btrix-admin" }}
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentd
+  namespace: {{ .Values.logging.namespace | default "btrix-admin" }}
+  labels:
+    k8s-app: fluentd-logging
+    version: v1
+spec:
+  selector:
+    matchLabels:
+      k8s-app: fluentd-logging
+      version: v1
+  template:
+    metadata:
+      labels:
+        k8s-app: fluentd-logging
+        version: v1
+    spec:
+      serviceAccount: fluentd
+      serviceAccountName: fluentd
+      tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      - key: "nodeType"
+        operator: "Equal"
+        value: "crawling"
+        effect: "NoSchedule"
+      containers:
+      - name: fluentd
+        image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch
+        env:
+          - name: K8S_NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name:  FLUENT_ELASTICSEARCH_HOST
+            value: "btrixlog-es-http.{{ .Values.logging.namespace | default "btrix-admin" }}.svc.cluster.local"
+          - name:  FLUENT_ELASTICSEARCH_PORT
+            value: "9200"
+          - name: FLUENT_ELASTICSEARCH_SCHEME
+            value: "https"
+          # Option to configure elasticsearch plugin with self signed certs
+          # ================================================================
+          - name: FLUENT_ELASTICSEARCH_SSL_VERIFY
+            value: "false"
+          # Option to configure elasticsearch plugin with tls
+          # ================================================================
+          - name: FLUENT_ELASTICSEARCH_SSL_VERSION
+            value: "TLSv1_2"
+          - name: FLUENTD_SYSTEMD_CONF
+            value: 'disable'
+          # X-Pack Authentication
+          # =====================
+          - name: FLUENT_ELASTICSEARCH_USER
+            value: "elastic"
+          - name: FLUENT_ELASTICSEARCH_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: btrixlog-es-elastic-user
+                key: elastic
+          # =====================
+          - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH
+            value: /var/log/containers/fluent*
+          # - name: FLUENT_CONTAINER_TAIL_PARSER_TYPE
+          #   value: /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<log>.*)$/
+        resources:
+          limits:
+            cpu: {{ .Values.logging.fluentd.cpu | default "60m" }}
+            memory: {{ .Values.logging.fluentd.mem | default "200Mi" }}
+          requests:
+            cpu: {{ .Values.logging.fluentd.cpu | default "60m" }}
+            memory: {{ .Values.logging.fluentd.mem | default "200Mi" }}
+        volumeMounts:
+        - name: varlog
+          mountPath: {{ .Values.logging.fluentd.logVar | default "/var/log" }}
+        # When actual pod logs in /var/lib/docker/containers, the following lines should be used.
+        - name: dockercontainerlogdirectory
+          mountPath: {{ .Values.logging.fluentd.logPathContainers | default "/var/lib/docker/containers" }}
+          readOnly: true
+        # When actual pod logs in /var/log/pods, the following lines should be used.
+        # - name: dockercontainerlogdirectory
+        #   mountPath: /var/log/pods
+        #   readOnly: true
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: {{ .Values.logging.fluentd.logVar | default "/var/log" }}
+      # When actual pod logs in /var/lib/docker/containers, the following lines should be used.
+      - name: dockercontainerlogdirectory
+        hostPath:
+          path: {{ .Values.logging.fluentd.logPathContainers | default "/var/lib/docker/containers" }}
+      # When actual pod logs in /var/log/pods, the following lines should be used.
+      # - name: dockercontainerlogdirectory
+      #   hostPath:
+      #     path: /var/log/pods
+{{- end -}}
--- a/chart/admin/logging/templates/ingress.yaml
+++ b/chart/admin/logging/templates/ingress.yaml
@ -0,0 +1,42 @@
+{{- define "ingress.install" -}}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: kibana-main
+  namespace: {{ .Values.logging.namespace | default "btrix-admin" }}
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/rewrite-target: /$1
+    nginx.ingress.kubernetes.io/enable-cors: "true"
+    nginx.ingress.kubernetes.io/cors-allow-origin: "*"
+    {{ if .Values.logging.ingress.tls }}
+    cert-manager.io/cluster-issuer: "cert-main"
+    {{ end }}
+    nginx.ingress.kubernetes.io/proxy-connect-timeout: "30"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "1800"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "1800"
+    nginx.ingress.kubernetes.io/upstream-vhost: "{{ .Values.logging.ingress.host | default "localhost" }}"
+    nginx.ingress.kubernetes.io/backend-protocol: "https"
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header X-Forwarded-Proto "https";
+spec:
+  {{ if .Values.logging.ingress.tls }}
+  tls:
+    - hosts:
+      - {{ .Values.logging.ingress.host | default "localhost" }}
+      secretName: cert-main
+  {{ end }}
+  rules:
+  - host: {{ .Values.logging.ingress.host | default "localhost" }}
+    http:
+      paths:
+      - path: {{ .Values.logging.ingress.path | default "/kibana" }}/(.*)
+        pathType: Prefix
+        backend:
+          service:
+            name: btrixlog-kb-http
+            port:
+              number: 5601
+{{- end -}}
--- a/chart/admin/logging/templates/kibana.yaml
+++ b/chart/admin/logging/templates/kibana.yaml
@ -0,0 +1,45 @@
+{{- define "kb.install" -}}
+---
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: btrixlog
+  namespace: {{ .Values.logging.namespace | default "btrix-admin" }}
+spec:
+  version: 8.5.3
+  count: 1
+  elasticsearchRef:
+    name: btrixlog
+  config:
+    server:
+      basePath: "{{ .Values.logging.ingress.path | default "/kibana" }}"
+      rewriteBasePath: false
+      publicBaseUrl: "https://{{ .Values.logging.ingress.host | default "localhost" }}{{ .Values.logging.ingress.path | default "/kibana" }}"
+  podTemplate:
+    spec:
+      containers:
+      - name: kibana
+        {{ if not .Values.logging.kibana.local }}
+        env:
+          - name: NODE_OPTIONS
+            value: "{{ .Values.logging.kibana.opt | default "--max-old-space-size=1024" }}"
+        resources:
+          requests:
+            memory: {{ .Values.logging.kibana.mem | default "1Gi" }}
+            cpu: {{ .Values.logging.kibana.cpu | default "1" }}
+          limits:
+            memory: {{ .Values.logging.kibana.mem | default "1Gi" }}
+            cpu: {{ .Values.logging.kibana.cpu | default "1" }}
+        {{ end }}
+      {{ if .Values.logging.dedicatedNode.enabled }}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: nodeType
+                operator: In
+                values:
+                - "{{ .Values.logging.dedicatedNode.nodeType | default "admin" }}"
+      {{ end }}
+{{- end -}}
--- a/chart/admin/logging/templates/logging.yaml
+++ b/chart/admin/logging/templates/logging.yaml
@ -0,0 +1,8 @@
+{{ if .Values.logging.enabled }}
+
+{{ include "es.install" . }}
+{{ include "kb.install" . }}
+{{ include "ingress.install" . }}
+{{ include "fluentd.install" . }}
+
+{{ end }}
--- a/chart/admin/logging/values.yaml
+++ b/chart/admin/logging/values.yaml
@ -0,0 +1,27 @@
+logging:
+  namespace: btrix-admin
+  enabled: true
+  dedicatedNode:
+    enabled: false
+    nodeType: admin
+  ingress:
+    tls: false
+    host: localhost
+    path: /kibana
+  elasticsearch:
+    local: true
+    cpu: 1
+    mem: 4Gi
+    opt: -Xms2g -Xmx2g
+    volumeEnabled: false
+    volumeSize: 1Gi
+  kibana:
+    local: true
+    cpu: 1
+    mem: 1Gi
+    opt: --max-old-space-size=1024
+  fluentd:
+    logVar: /var/log
+    logPathContainers: /var/lib/docker/containers
+    cpu: 60m
+    mem: 200Mi
--- a/chart/charts/btrix-admin-logging-0.1.0.tgz
+++ b/chart/charts/btrix-admin-logging-0.1.0.tgz
--- a/chart/examples/local-logging.yaml
+++ b/chart/examples/local-logging.yaml
@ -0,0 +1,23 @@
+ingress:
+  host: "myhostname"
+  scheme: "https"
+  tls: false
+
+ingress_class: "nginx"
+
+addons:
+  admin:
+    logging: true
+
+btrix-admin-logging:
+  logging:
+    enabled: true
+    dedicatedNode:
+      enabled: false
+    ingress:
+      tls: false
+      host: "myhostname"
+    elasticsearch:
+      local: true
+    kibana:
+      local: true
--- a/chart/values.yaml
+++ b/chart/values.yaml
@ -225,3 +225,9 @@ signer_limits_memory: "96Mi"
 #   annotations:
 #     service.beta.kubernetes.io/aws-load-balancer-internal: "true"
 #     helm.sh/resource-policy: keep
+
+# Admin services (see Chart.yaml's dependencies)
+# note: see `chart/examples/local-logging.yaml`
+addons:
+  admin:
+    logging: false