diff --git a/chart/Chart.lock b/chart/Chart.lock new file mode 100644 index 00000000..80c95721 --- /dev/null +++ b/chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: btrix-admin-logging + repository: file://./admin/logging + version: 0.1.0 +digest: sha256:a62579385835818c52ddc8c59d6ec5a17800e60b71a2e75b3a85a02c262bb385 +generated: "2023-01-03T00:22:20.096291-08:00" diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 13de7909..d51543d5 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -14,3 +14,8 @@ version: 0.1.0 # follow Semantic Versioning. They should reflect the version the application is using. appVersion: 0.1.0 +dependencies: + - name: btrix-admin-logging + version: 0.1.0 + condition: addons.admin.logging + repository: file://./admin/logging diff --git a/chart/README.md b/chart/README.md new file mode 100644 index 00000000..79bd69ba --- /dev/null +++ b/chart/README.md @@ -0,0 +1,7 @@ +## Update Helm dependencies + +* It needs to update Helm charts after changing its dependencies (e.g. logging) + +``` +$ helm dependency update . +``` diff --git a/chart/admin/logging/.gitignore b/chart/admin/logging/.gitignore new file mode 100644 index 00000000..58add82d --- /dev/null +++ b/chart/admin/logging/.gitignore @@ -0,0 +1,2 @@ +old +_tmp_prod_ diff --git a/chart/admin/logging/Chart.yaml b/chart/admin/logging/Chart.yaml new file mode 100644 index 00000000..3a0a8500 --- /dev/null +++ b/chart/admin/logging/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: btrix-admin-logging +description: A chart for running the Webrecorder Browsertrix System - admin services +type: application +icon: https://webrecorder.net/assets/icon.png + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 0.1.0 diff --git a/chart/admin/logging/README.md b/chart/admin/logging/README.md new file mode 100644 index 00000000..0dadc17e --- /dev/null +++ b/chart/admin/logging/README.md @@ -0,0 +1,74 @@ +# Btrix Logging Service + +## Prerequisites + +1. [optional] add a new node for a dedicated logging stacks (Elasticsearch, Fluentd, Kibana) +1. [optional] set a label for a node with `NodeType=admin` (when `dedicatedNode` is `true`) +``` +kubectl label nodes new-admin-node nodeType=admin +``` +1. edit `chart/values.yaml` to enable `logging` service when install with browsestrix-cloud +``` +addons: + admin: + logging: true +``` +This will enable the helm dependencies defined in `chart/Chart.yaml`. + +And, edit `chart/examples/local-logging.yaml` for a local test. + +Optionally, when install the logging service only, edit `chart/admin/logging/values.yaml`. +For a local test, it should use a hostname (not `localhost` but a hostname registered in `/etc/hosts`) + +## Installation + +* run a setup script (will create a namespace and install elastic's CRDS) +``` +$ ./chart/admin/logging/scripts/eck_install.sh +``` +* install logging services using helm chart +``` +helm upgrade --install -f ./chart/values.yaml -f ./chart/examples/local-logging.yaml btrix ./chart +``` + +## Installation (logging service only) + +* run a setup script (will create a namespace and install elastic's CRDS) +``` +$ ./chart/admin/logging/scripts/eck_install.sh +``` +* install logging services using helm chart +``` +helm upgrade --install -f ./chart/admin/logging/values.yaml btrix-admin-log ./chart/admin/logging +``` + +## Access Kibana dashboard + +* get the Kibana's login password (username: `elastic`) +``` +kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}' +``` +* open `https://hostname/kibana` + +## Import/Export Kibana data + +* Import data (e.g. data view, search queries and dashboards) + +``` +cd ./chart/admin/logging/scripts +./kibana_imports.sh +``` + +* Exports data (e.g. data view, search queries and dashboards) + +``` +cd ./chart/admin/logging/scripts +./kibana_exports.sh +``` + +## Uninstallation + +``` +$ helm uninstall btrix-admin-log +$ ./chart/admin/logging/scripts/eck_uninstall.sh +``` diff --git a/chart/admin/logging/scripts/eck_install.sh b/chart/admin/logging/scripts/eck_install.sh new file mode 100755 index 00000000..a1f286c8 --- /dev/null +++ b/chart/admin/logging/scripts/eck_install.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +kubectl create namespace btrix-admin +kubectl create -f https://download.elastic.co/downloads/eck/2.5.0/crds.yaml +kubectl apply -f https://download.elastic.co/downloads/eck/2.5.0/operator.yaml + +kubectl label nodes docker-desktop nodeType=admin +kubectl get nodes -o wide -o jsonpath='{.items[*].metadata.labels}' | jq . diff --git a/chart/admin/logging/scripts/eck_uninstall.sh b/chart/admin/logging/scripts/eck_uninstall.sh new file mode 100755 index 00000000..3a9e5696 --- /dev/null +++ b/chart/admin/logging/scripts/eck_uninstall.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +kubectl delete -f https://download.elastic.co/downloads/eck/2.5.0/operator.yaml +kubectl delete -f https://download.elastic.co/downloads/eck/2.5.0/crds.yaml +kubectl delete namespace btrix-admin diff --git a/chart/admin/logging/scripts/kibana_export.ndjson b/chart/admin/logging/scripts/kibana_export.ndjson new file mode 100644 index 00000000..9efd90e1 --- /dev/null +++ b/chart/admin/logging/scripts/kibana_export.ndjson @@ -0,0 +1,8 @@ +{"attributes":{"fieldAttrs":"{\"kubernetes.host\":{\"count\":2},\"kubernetes.labels.app\":{\"count\":2},\"kubernetes.pod_name\":{\"count\":3},\"log\":{\"count\":5},\"tag\":{\"count\":2}}","fieldFormatMap":"{}","fields":"[]","name":"btrix-cloud","runtimeFieldMap":"{}","sourceFilters":"[]","timeFieldName":"@timestamp","title":"logstash-*","typeMeta":"{}"},"coreMigrationVersion":"8.5.3","id":"4af04743-fb92-454c-9e13-75dc6422e01b","migrationVersion":{"index-pattern":"8.0.0"},"references":[],"type":"index-pattern","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2NSwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"8.5.3\",\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":15,\"i\":\"7447c31f-5270-437a-a849-aaad2c524bc3\"},\"panelIndex\":\"7447c31f-5270-437a-a849-aaad2c524bc3\",\"embeddableConfig\":{\"attributes\":{\"title\":\"\",\"visualizationType\":\"lnsXY\",\"type\":\"lens\",\"references\":[{\"type\":\"index-pattern\",\"id\":\"4af04743-fb92-454c-9e13-75dc6422e01b\",\"name\":\"indexpattern-datasource-layer-f89b17c5-99c7-4348-972f-b072475a449f\"}],\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"None\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"f89b17c5-99c7-4348-972f-b072475a449f\",\"accessors\":[\"0a3522e2-426b-4246-a506-b8dddf6ae0e2\"],\"position\":\"top\",\"seriesType\":\"line\",\"showGridlines\":false,\"layerType\":\"data\",\"xAccessor\":\"293a66d9-5cb2-46f2-b204-99b2d91f693b\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"indexpattern\":{\"layers\":{\"f89b17c5-99c7-4348-972f-b072475a449f\":{\"columns\":{\"293a66d9-5cb2-46f2-b204-99b2d91f693b\":{\"label\":\"@timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"@timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"0a3522e2-426b-4246-a506-b8dddf6ae0e2\":{\"label\":\"Count of records\",\"customLabel\":false,\"dataType\":\"number\",\"operationType\":\"count\",\"isBucketed\":false,\"scale\":\"ratio\",\"sourceField\":\"___records___\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"293a66d9-5cb2-46f2-b204-99b2d91f693b\",\"0a3522e2-426b-4246-a506-b8dddf6ae0e2\"],\"incompleteColumns\":{}}}}},\"internalReferences\":[],\"adHocDataViews\":{}}},\"enhancements\":{},\"hidePanelTitles\":false},\"title\":\"Graph\"}]","timeRestore":false,"title":"btrix-cloud","version":1},"coreMigrationVersion":"8.5.3","id":"b2460db0-8629-11ed-a271-e3c0b9803788","migrationVersion":{"dashboard":"8.5.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"7447c31f-5270-437a-a849-aaad2c524bc3:indexpattern-datasource-layer-f89b17c5-99c7-4348-972f-b072475a449f","type":"index-pattern"}],"type":"dashboard","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2NiwxXQ=="} +{"attributes":{"columns":["log","kubernetes.pod_name"],"description":"","grid":{"columns":{"log":{"width":1061}}},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.namespace_name.keyword: crawlers AND ERROR\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-crawlers-errors","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"c08d4250-861d-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2NywxXQ=="} +{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.pod_name.keyword: job-*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-crawl-jobs","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"c132b4a0-861e-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2OCwxXQ=="} +{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.namespace_name.keyword: crawlers\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-crawlers","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"645f4720-861e-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk2OSwxXQ=="} +{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.pod_name.keyword : browsertrix-cloud-frontend-*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-frontend","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"04c7feb0-861e-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk3MCwxXQ=="} +{"attributes":{"columns":["log"],"description":"","grid":{},"hideChart":false,"isTextBasedQuery":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"kubernetes.pod_name.keyword : browsertrix-cloud-backend-*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"timeRestore":false,"title":"Btrix-backend","usesAdHocDataView":false},"coreMigrationVersion":"8.5.3","id":"f3cf5c20-861d-11ed-a271-e3c0b9803788","migrationVersion":{"search":"8.0.0"},"references":[{"id":"4af04743-fb92-454c-9e13-75dc6422e01b","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2022-12-27T21:45:50.441Z","version":"Wzk3MSwxXQ=="} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":7,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/chart/admin/logging/scripts/kibana_exports.sh b/chart/admin/logging/scripts/kibana_exports.sh new file mode 100755 index 00000000..a4203547 --- /dev/null +++ b/chart/admin/logging/scripts/kibana_exports.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +ES_USER="elastic" +ES_PASS=$(kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}') +KIBANA_INGRESS="kibana-main" +HOSTNAME=`kubectl get ingress -A | grep $KIBANA_INGRESS | awk '{print $4}'` +KIBANA_URL="https://${HOSTNAME}/kibana" +EXPORT_FN="./kibana_export.ndjson" + +echo "use $KIBANA_URL" + +RET=`curl --user $ES_USER:$ES_PASS \ + "${KIBANA_URL}/api/kibana/management/saved_objects/_find?perPage=50&page=1&fields=id&type=url&type=index-pattern&type=action&type=query&type=alert&type=search&type=graph-workspace&type=tag&type=csp_rule&type=csp-rule-template&type=visualization&type=canvas-element&type=canvas-workpad&type=dashboard&type=lens&type=map&type=cases&type=osquery-saved-query&type=osquery-pack&type=uptime-dynamic-settings&type=synthetics-privates-locations&type=infrastructure-ui-source&type=metrics-explorer-view&type=inventory-view&type=infrastructure-monitoring-log-view&type=apm-indices&sortField=updated_at&sortOrder=desc" | \ + jq -r ".saved_objects | [.[] | { id:.id, type:.type}] | @json"` + +curl --user $ES_USER:$ES_PASS -X POST \ + "${KIBANA_URL}/api/saved_objects/_export" \ + -H 'kbn-xsrf: true' \ + -H 'Content-Type: application/json' \ + -d "{\"objects\": $RET }" > $EXPORT_FN diff --git a/chart/admin/logging/scripts/kibana_imports.sh b/chart/admin/logging/scripts/kibana_imports.sh new file mode 100755 index 00000000..ea12c1ab --- /dev/null +++ b/chart/admin/logging/scripts/kibana_imports.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +ES_USER="elastic" +ES_PASS=$(kubectl get secret btrixlog-es-elastic-user -n btrix-admin -o go-template='{{.data.elastic | base64decode}}') +KIBANA_INGRESS="kibana-main" +HOSTNAME=`kubectl get ingress -A | grep $KIBANA_INGRESS | awk '{print $4}'` +KIBANA_URL="https://${HOSTNAME}/kibana" +EXPORT_FN="./kibana_export.ndjson" + +echo "use $KIBANA_URL" + +curl -k --user $ES_USER:$ES_PASS -X POST \ + "${KIBANA_URL}/api/saved_objects/_import" \ + -H "kbn-xsrf: true" \ + --form file=@$EXPORT_FN diff --git a/chart/admin/logging/templates/elasticsearch.yaml b/chart/admin/logging/templates/elasticsearch.yaml new file mode 100644 index 00000000..b028e354 --- /dev/null +++ b/chart/admin/logging/templates/elasticsearch.yaml @@ -0,0 +1,70 @@ +{{- define "es.install" -}} +{{ if .Values.logging.elasticsearch.volumeEnabled }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: es-storage-pvc + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} + annotations: + "helm.sh/resource-policy": keep +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.logging.elasticsearch.volumeSize | default "1Gi" }} + storageClassName: hostpath +{{ end }} +--- +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: btrixlog + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} +spec: + version: 8.5.3 + nodeSets: + - name: default + count: 1 + config: + node.store.allow_mmap: false + podTemplate: + spec: + {{ if .Values.logging.elasticsearch.volumeEnabled }} + volumes: + - name: data-es + persistentVolumeClaim: + claimName: es-storage-pvc + {{ end }} + {{ if .Values.logging.dedicatedNode.enabled }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nodeType + operator: In + values: + - "{{ .Values.logging.dedicatedNode.nodeType | default "admin" }}" + {{ end }} + containers: + - name: elasticsearch + {{ if not .Values.logging.elasticsearch.local }} + env: + - name: ES_JAVA_OPTS + value: "{{ .Values.logging.elasticsearch.opt | default "-Xms2g -Xmx2g" }}" + resources: + requests: + memory: {{ .Values.logging.elasticsearch.mem | default "4Gi" }} + cpu: {{ .Values.logging.elasticsearch.cpu | default "1" }} + limits: + memory: {{ .Values.logging.elasticsearch.mem | default "4Gi" }} + cpu: {{ .Values.logging.elasticsearch.cpu | default "1" }} + {{ end }} + {{ if .Values.logging.elasticsearch.volumeEnabled }} + volumeMounts: + - name: data-es + mountPath: /usr/share/elasticsearch/data + {{ end }} +{{- end -}} diff --git a/chart/admin/logging/templates/fluentd.yaml b/chart/admin/logging/templates/fluentd.yaml new file mode 100644 index 00000000..8d9ac878 --- /dev/null +++ b/chart/admin/logging/templates/fluentd.yaml @@ -0,0 +1,138 @@ +{{- define "fluentd.install" -}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluentd + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fluentd +rules: +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - get + - list + - watch + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fluentd +roleRef: + kind: ClusterRole + name: fluentd + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: fluentd + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentd + namespace: {{ .Values.logging.namespace | default "btrix-admin" }} + labels: + k8s-app: fluentd-logging + version: v1 +spec: + selector: + matchLabels: + k8s-app: fluentd-logging + version: v1 + template: + metadata: + labels: + k8s-app: fluentd-logging + version: v1 + spec: + serviceAccount: fluentd + serviceAccountName: fluentd + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: "nodeType" + operator: "Equal" + value: "crawling" + effect: "NoSchedule" + containers: + - name: fluentd + image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: FLUENT_ELASTICSEARCH_HOST + value: "btrixlog-es-http.{{ .Values.logging.namespace | default "btrix-admin" }}.svc.cluster.local" + - name: FLUENT_ELASTICSEARCH_PORT + value: "9200" + - name: FLUENT_ELASTICSEARCH_SCHEME + value: "https" + # Option to configure elasticsearch plugin with self signed certs + # ================================================================ + - name: FLUENT_ELASTICSEARCH_SSL_VERIFY + value: "false" + # Option to configure elasticsearch plugin with tls + # ================================================================ + - name: FLUENT_ELASTICSEARCH_SSL_VERSION + value: "TLSv1_2" + - name: FLUENTD_SYSTEMD_CONF + value: 'disable' + # X-Pack Authentication + # ===================== + - name: FLUENT_ELASTICSEARCH_USER + value: "elastic" + - name: FLUENT_ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: btrixlog-es-elastic-user + key: elastic + # ===================== + - name: FLUENT_CONTAINER_TAIL_EXCLUDE_PATH + value: /var/log/containers/fluent* + # - name: FLUENT_CONTAINER_TAIL_PARSER_TYPE + # value: /^(?