diff --git a/.github/workflows/k3d-log-ci.yaml b/.github/workflows/k3d-log-ci.yaml new file mode 100644 index 00000000..f9cd83a0 --- /dev/null +++ b/.github/workflows/k3d-log-ci.yaml @@ -0,0 +1,83 @@ +name: Cluster Run (K3d-Admin-Logging) + +on: [push, pull_request] + +jobs: + btrix-k3d-admin-logging-test: + runs-on: ubuntu-latest + steps: + - name: Create k3d Cluster + uses: AbsaOSS/k3d-action@v2 + with: + cluster-name: btrix-1 + args: >- + --agents 1 + -p "443:443@loadbalancer" + --k3s-arg "--no-deploy=traefik,metrics-server@server:*" + + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Kubectl + uses: azure/setup-kubectl@v3 + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: 3.10.2 + + - name: Add hosts to /etc/hosts + run: | + sudo echo "127.0.0.1 myhost.internal" | sudo tee -a /etc/hosts + + - name: Install nginx ingress controller + run: | + helm install --namespace kube-system nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx + sleep 5 + POD=$(kubectl get pods -A | grep nginx-ingress | head -n 1 | awk '{print $2}') + while [[ $(kubectl get pods $POD -n kube-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do kubectl get pods -n kube-system; sleep 5; done + sleep 20 + + - name: Start Cluster with Helm (chart/admin/logging) + run: | + ./chart/admin/logging/scripts/eck_install.sh + helm upgrade --install -f ./chart/admin/logging/values.yaml btrix-admin-log ./chart/admin/logging --set logging.ingress.host=myhost.internal + + - name: Wait for all pods to be ready + run: | + # waiting fluentd + kubectl wait --for=condition=ready pod --all --timeout=200s -n btrix-admin + # waiting elastic-operator and ES/Kibana + while [[ $(kubectl get pods -n elastic-system -l control-plane=elastic-operator -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do kubectl get pods -n elastic-system; sleep 5; done + while [[ $(kubectl get pods -n btrix-admin -l common.k8s.elastic.co/type=elasticsearch -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do kubectl get pods -n btrix-admin; sleep 5; done + while [[ $(kubectl get pods -n btrix-admin -l common.k8s.elastic.co/type=kibana -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do kubectl get pods -n btrix-admin; sleep 5; done + kubectl get pod -n btrix-admin -o wide + kubectl get svc -n btrix-admin -o wide + kubectl get ingress -A -o wide + kubectl describe ingress kibana-main -n btrix-admin + # check the auto-generated password for elastic + while kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}'; ret=$?; [ $ret -ne 0 ]; do sleep 5; done + + - name: Test Kibana via port-forward + run: | + sleep 20 + kubectl port-forward service/btrixlog-kb-http -n btrix-admin 5601:5601 & + sleep 5 + curl -k -v https://myhost.internal:5601/kibana/ 2>&1 | grep -i "location" + + - name: Test Kibana via Ingress + run: | + kubectl get ingress -A -o wide + # kubectl describe ingress kibana-main -n btrix-admin + # kubectl describe pod $POD -n kube-system + curl -v -k https://myhost.internal/kibana/ 2>&1 | grep -i "location" + + - name: Install Kibana dashboard + run: | + cd ./chart/admin/logging/scripts + ./kibana_imports.sh + + - name: Clean up + run: | + helm uninstall btrix-admin-log + ./chart/admin/logging/scripts/eck_uninstall.sh diff --git a/chart/Chart.lock b/chart/Chart.lock new file mode 100644 index 00000000..80c95721 --- /dev/null +++ b/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: btrix-admin-logging + repository: file://./admin/logging + version: 0.1.0 +digest: sha256:a62579385835818c52ddc8c59d6ec5a17800e60b71a2e75b3a85a02c262bb385 +generated: "2023-01-03T00:22:20.096291-08:00" diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 13de7909..d51543d5 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -14,3 +14,8 @@ version: 0.1.0 # follow Semantic Versioning. They should reflect the version the application is using. appVersion: 0.1.0 +dependencies: + - name: btrix-admin-logging + version: 0.1.0 + condition: addons.admin.logging + repository: file://./admin/logging diff --git a/chart/README.md b/chart/README.md new file mode 100644 index 00000000..79bd69ba --- /dev/null +++ b/chart/README.md @@ -0,0 +1,7 @@ +## Update Helm dependencies + +* It needs to update Helm charts after changing its dependencies (e.g. logging) + +``` +$ helm dependency update . +``` diff --git a/chart/admin/logging/.gitignore b/chart/admin/logging/.gitignore new file mode 100644 index 00000000..58add82d --- /dev/null +++ b/chart/admin/logging/.gitignore @@ -0,0 +1,2 @@ +old +_tmp_prod_ diff --git a/chart/admin/logging/Chart.yaml b/chart/admin/logging/Chart.yaml new file mode 100644 index 00000000..3a0a8500 --- /dev/null +++ b/chart/admin/logging/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: btrix-admin-logging +description: A chart for running the Webrecorder Browsertrix System - admin services +type: application +icon: https://webrecorder.net/assets/icon.png + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 0.1.0 diff --git a/chart/admin/logging/README.md b/chart/admin/logging/README.md new file mode 100644 index 00000000..3d154828 --- /dev/null +++ b/chart/admin/logging/README.md @@ -0,0 +1,74 @@ +# Btrix Logging Service + +## Prerequisites + +1. [optional] add a new node for a dedicated logging stacks (Elasticsearch, Fluentd, Kibana) +1. [optional] set a label for a node with `NodeType=admin` (when `dedicatedNode` is `true`) +``` +kubectl label nodes new-admin-node nodeType=admin +``` +1. edit `chart/values.yaml` to enable `logging` service when install with browsestrix-cloud +``` +addons: + admin: + logging: true +``` +This will enable the helm dependencies defined in `chart/Chart.yaml`. + +And, edit `chart/examples/local-logging.yaml` for a local test. + +Optionally, when install the logging service only, edit `chart/admin/logging/values.yaml`. +For a local test, it should use a hostname (not `localhost` but a hostname like `myhostname` registered in `/etc/hosts`) + +## Installation + +* run a setup script (will create a namespace and install elastic's CRDS) +``` +$ ./chart/admin/logging/scripts/eck_install.sh +``` +* install logging services using helm chart +``` +helm upgrade --install -f ./chart/values.yaml -f ./chart/examples/local-logging.yaml btrix ./chart +``` + +## Installation (logging service only) + +* run a setup script (will create a namespace and install elastic's CRDS) +``` +$ ./chart/admin/logging/scripts/eck_install.sh +``` +* install logging services using helm chart +``` +helm upgrade --install -f ./chart/admin/logging/values.yaml btrix-admin-log ./chart/admin/logging +``` + +## Access Kibana dashboard + +* get the Kibana's login password (username: `elastic`) +``` +kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}' +``` +* open `https://hostname/kibana` + +## Import/Export Kibana data + +* Import data (e.g. data view, search queries and dashboards) + +``` +cd ./chart/admin/logging/scripts +./kibana_imports.sh +``` + +* Exports data (e.g. data view, search queries and dashboards) + +``` +cd ./chart/admin/logging/scripts +./kibana_exports.sh +``` + +## Uninstallation + +``` +$ helm uninstall btrix-admin-log +$ ./chart/admin/logging/scripts/eck_uninstall.sh +``` diff --git a/chart/admin/logging/scripts/eck_install.sh b/chart/admin/logging/scripts/eck_install.sh new file mode 100755 index 00000000..1005359c --- /dev/null +++ b/chart/admin/logging/scripts/eck_install.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +kubectl create namespace btrix-admin +kubectl create -f https://download.elastic.co/downloads/eck/2.5.0/crds.yaml +kubectl apply -f https://download.elastic.co/downloads/eck/2.5.0/operator.yaml + +# kubectl label nodes docker-desktop nodeType=admin +kubectl get nodes +kubectl get nodes -o wide -o jsonpath='{.items[*].metadata.labels}' | jq . diff --git a/chart/admin/logging/scripts/eck_uninstall.sh b/chart/admin/logging/scripts/eck_uninstall.sh new file mode 100755 index 00000000..3a9e5696 --- /dev/null +++ b/chart/admin/logging/scripts/eck_uninstall.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +kubectl delete -f https://download.elastic.co/downloads/eck/2.5.0/operator.yaml +kubectl delete -f https://download.elastic.co/downloads/eck/2.5.0/crds.yaml +kubectl delete namespace btrix-admin diff --git a/chart/admin/logging/scripts/kibana_export.ndjson b/chart/admin/logging/scripts/kibana_export.ndjson new file mode 100644 index 00000000..9efd90e1 --- /dev/null +++ b/chart/admin/logging/scripts/kibana_export.ndjson @@ -0,0 +1,8 @@ +{"attributes":{"fieldAttrs":"{\"kubernetes.host\":{\"count\":2},\"kubernetes.labels.app\":{\"count\":2},\"kubernetes.pod_name\":{\"count\":3},\"log\":{\"count\":5},\"tag\":{\"count\":2}}","fieldFormatMap":"{}","fields":"[]","name":"btrix-cloud","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"@timestamp","title":"logstash-*","typeMeta":"{}"},"coreMigrationVersion":"8.5.3","id":"4af04743-fb92-454c-9e13-75dc6422e01b","migrationVersion":{"index-pattern":"8.0.0"},"references":[],"type":"index-pattern","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2NSwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"8.5.3\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":15,\"i\":\"7447c31f-5270-437a-a849-aaad2c524bc3\"},\"panelIndex\":\"7447c31f-5270-437a-a849-aaad2c524bc3\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"4af04743-fb92-454c-9e13-75dc6422e01b\",\"name\":\"indexpattern-datasource-layer-f89b17c5-99c7-4348-972f-b072475a449f\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"f89b17c5-99c7-4348-972f-b072475a449f\",\"accessors\":[\"0a3522e2-426b-4246-a506-b8dddf6ae0e2\"],\"position\":\"top\",\"seriesType\":\"line\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"293a66d9-5cb2-46f2-b204-99b2d91f693b\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"f89b17c5-99c7-4348-972f-b072475a449f\":{\"columns\":{\"293a66d9-5cb2-46f2-b204-99b2d91f693b\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"0a3522e2-426b-4246-a506-b8dddf6ae0e2\":{\"label\":\"Count of records\",\"customLabel\":false,\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"293a66d9-5cb2-46f2-b204-99b2d91f693b\",\"0a3522e2-426b-4246-a506-b8dddf6ae0e2\"],\"incompleteColumns\":{}}}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Graph\"}]","timeRestore":false,"title":"btrix-cloud","version":1},"coreMigrationVersion":"8.5.3","id":"b2460db0-8629-11ed-a271-e3c0b9803788","migrationVersion":{"dashboard":"8.5.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"7447c31f-5270-437a-a849-aaad2c524bc3:indexpattern-datasource-layer-f89b17c5-99c7-4348-972f-b072475a449f","type":"index-pattern"}],"type":"dashboard","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2NiwxXQ=="} +{"attributes":{"columns":["log","kubernetes.pod_name"],"description":"","grid":{"columns":{"log":{"width":1061}}},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.namespace_name.keyword: crawlers AND ERROR\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-crawlers-errors","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"c08d4250-861d-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2NywxXQ=="} +{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.pod_name.keyword: job-*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-crawl-jobs","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"c132b4a0-861e-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2OCwxXQ=="} +{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.namespace_name.keyword: crawlers\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-crawlers","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"645f4720-861e-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2OSwxXQ=="} +{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.pod_name.keyword : browsertrix-cloud-frontend-*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-frontend","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"04c7feb0-861e-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk3MCwxXQ=="} +{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.pod_name.keyword : browsertrix-cloud-backend-*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-backend","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"f3cf5c20-861d-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk3MSwxXQ=="} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":7,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/chart/admin/logging/scripts/kibana_exports.sh b/chart/admin/logging/scripts/kibana_exports.sh new file mode 100755 index 00000000..a4203547 --- /dev/null +++ b/chart/admin/logging/scripts/kibana_exports.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +ES_USER="elastic" +ES_PASS=$(kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}') +KIBANA_INGRESS="kibana-main" +HOSTNAME=`kubectl get ingress -A | grep $KIBANA_INGRESS | awk '{print $4}'` +KIBANA_URL="https://${HOSTNAME}/kibana" +EXPORT_FN="./kibana_export.ndjson" + +echo "use $KIBANA_URL" + +RET=`curl --user $ES_USER:$ES_PASS \ + "${KIBANA_URL}/api/kibana/management/saved_objects/_find?perPage=50&page=1&fields=id&type=url&type=index-pattern&type=action&type=query&type=alert&type=search&type=graph-workspace&type=tag&type=csp_rule&type=csp-rule-template&type=visualization&type=canvas-element&type=canvas-workpad&type=dashboard&type=lens&type=map&type=cases&type=osquery-saved-query&type=osquery-pack&type=uptime-dynamic-settings&type=synthetics-privates-locations&type=infrastructure-ui-source&type=metrics-explorer-view&type=inventory-view&type=infrastructure-monitoring-log-view&type=apm-indices&sortField=updated_at&sortOrder=desc" | \ + jq -r ".saved_objects | [.[] | { id:.id, type:.type}] | @json"` + +curl --user $ES_USER:$ES_PASS -X POST \ + "${KIBANA_URL}/api/saved_objects/_export" \ + -H 'kbn-xsrf: true' \ + -H 'Content-Type: application/json' \ + -d "{\"objects\": $RET }" > $EXPORT_FN diff --git a/chart/admin/logging/scripts/kibana_imports.sh b/chart/admin/logging/scripts/kibana_imports.sh new file mode 100755 index 00000000..ea12c1ab --- /dev/null +++ b/chart/admin/logging/scripts/kibana_imports.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +ES_USER="elastic" +ES_PASS=$(kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}') +KIBANA_INGRESS="kibana-main" +HOSTNAME=`kubectl get ingress -A | grep $KIBANA_INGRESS | awk '{print $4}'` +KIBANA_URL="https://${HOSTNAME}/kibana" +EXPORT_FN="./kibana_export.ndjson" + +echo "use $KIBANA_URL" + +curl -k --user $ES_USER:$ES_PASS -X POST \ + "${KIBANA_URL}/api/saved_objects/_import" \ + -H "kbn-xsrf: true" \ + --form file=@$EXPORT_FN diff --git a/chart/admin/logging/templates/elasticsearch.yaml b/chart/admin/logging/templates/elasticsearch.yaml new file mode 100644 index 00000000..b028e354 --- /dev/null +++ b/chart/admin/logging/templates/elasticsearch.yaml @@ -0,0 +1,70 @@ +{{- define "es.install" -}} +{{ if .Values.logging.elasticsearch.volumeEnabled }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: es-storage-pvc + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} + annotations: + "helm.sh/resource-policy": keep +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.logging.elasticsearch.volumeSize | default "1Gi" }} + storageClassName: hostpath +{{ end }} +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: btrixlog + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} +spec: + version: 8.5.3 + nodeSets: + - name: default + count: 1 + config: + node.store.allow_mmap: false + podTemplate: + spec: + {{ if .Values.logging.elasticsearch.volumeEnabled }} + volumes: + - name: data-es + persistentVolumeClaim: + claimName: es-storage-pvc + {{ end }} + {{ if .Values.logging.dedicatedNode.enabled }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nodeType + operator: In + values: + - "{{ .Values.logging.dedicatedNode.nodeType | default "admin" }}" + {{ end }} + containers: + - name: elasticsearch + {{ if not .Values.logging.elasticsearch.local }} + env: + - name: ES_JAVA_OPTS + value: "{{ .Values.logging.elasticsearch.opt | default "-Xms2g -Xmx2g" }}" + resources: + requests: + memory: {{ .Values.logging.elasticsearch.mem | default "4Gi" }} + cpu: {{ .Values.logging.elasticsearch.cpu | default "1" }} + limits: + memory: {{ .Values.logging.elasticsearch.mem | default "4Gi" }} + cpu: {{ .Values.logging.elasticsearch.cpu | default "1" }} + {{ end }} + {{ if .Values.logging.elasticsearch.volumeEnabled }} + volumeMounts: + - name: data-es + mountPath: /usr/share/elasticsearch/data + {{ end }} +{{- end -}} diff --git a/chart/admin/logging/templates/fluentd.yaml b/chart/admin/logging/templates/fluentd.yaml new file mode 100644 index 00000000..7acdf27a --- /dev/null +++ b/chart/admin/logging/templates/fluentd.yaml @@ -0,0 +1,140 @@ +{{- define "fluentd.install" -}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluentd +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fluentd +roleRef: + kind: ClusterRole + name: fluentd + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: fluentd + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} + labels: + k8s-app: fluentd-logging + version: v1 +spec: + selector: + matchLabels: + k8s-app: fluentd-logging + version: v1 + template: + metadata: + labels: + k8s-app: fluentd-logging + version: v1 + spec: + serviceAccount: fluentd + serviceAccountName: fluentd + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: "nodeType" + operator: "Equal" + value: "crawling" + effect: "NoSchedule" + containers: + - name: fluentd + image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: FLUENT_ELASTICSEARCH_HOST + value: "btrixlog-es-http.{{ .Values.logging.namespace | default "btrix-admin" }}.svc.cluster.local" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + - name: FLUENT_ELASTICSEARCH_SCHEME + value: "https" + # Option to configure elasticsearch plugin with self signed certs + # ================================================================ + - name: FLUENT_ELASTICSEARCH_SSL_VERIFY + value: "false" + # Option to configure elasticsearch plugin with tls + # ================================================================ + - name: FLUENT_ELASTICSEARCH_SSL_VERSION + value: "TLSv1_2" + - name: FLUENTD_SYSTEMD_CONF + value: 'disable' + # X-Pack Authentication + # ===================== + - name: FLUENT_ELASTICSEARCH_USER + value: "elastic" + - name: FLUENT_ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: btrixlog-es-elastic-user + key: elastic + # ===================== + - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH + value: /var/log/containers/fluent* + {{ if not .Values.logging.kibana.local }} + - name: FLUENT_CONTAINER_TAIL_PARSER_TYPE + value: /^(?