tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

k8s: support for using a pre-made persistent volume/claim for crawling, configurable via CRAWLER_PV_CLAIM, otherwise using emptyDir

Browse Source 
k8s: ability to set deployment scale for frontend as well
This commit is contained in:
Ilya Kreymer 2022-03-15 11:18:23 -07:00
parent 912004751d
commit 4b2f89db91
4 changed files with 20 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/k8sman.py
View File

@ -56,6 +56,14 @@ class K8SManager:
self.requests_mem = os.environ["CRAWLER_REQUESTS_MEM"] self.requests_mem = os.environ["CRAWLER_REQUESTS_MEM"]
self.limits_mem = os.environ["CRAWLER_LIMITS_MEM"] self.limits_mem = os.environ["CRAWLER_LIMITS_MEM"]
self.crawl_volume = {"name": "crawl-data"}
# if set, use persist volume claim for crawls
crawl_pv_claim = os.environ.get("CRAWLER_PV_CLAIM")
if crawl_pv_claim:
self.crawl_volume["persistentVolumeClaim"] = {"claimName": crawl_pv_claim}
else:
self.crawl_volume["emptyDir"] = {}
self.loop = asyncio.get_running_loop() self.loop = asyncio.get_running_loop()
self.loop.create_task(self.run_event_loop()) self.loop.create_task(self.run_event_loop())
self.loop.create_task(self.init_redis(self.redis_url)) self.loop.create_task(self.init_redis(self.redis_url))
@ -742,7 +750,7 @@ class K8SManager:
], ],
}, },
}, },
{"name": "crawl-data", "emptyDir": {}}, self.crawl_volume,
], ],
"restartPolicy": "Never", "restartPolicy": "Never",
"terminationGracePeriodSeconds": self.grace_period, "terminationGracePeriodSeconds": self.grace_period,

4
chart/templates/configmap.yaml
View File

@ -21,6 +21,10 @@ data:
CRAWLER_REQUESTS_MEM: "{{ .Values.crawler_requests_memory }}" CRAWLER_REQUESTS_MEM: "{{ .Values.crawler_requests_memory }}"
CRAWLER_LIMITS_MEM: "{{ .Values.crawler_limits_memory }}" CRAWLER_LIMITS_MEM: "{{ .Values.crawler_limits_memory }}"
{{- if .Values.crawler_pv_claim }}
CRAWLER_PV_CLAIM: "{{ .Values.crawler_pv_claim }}"
{{- end }}
REDIS_URL: "{{ .Values.redis_url }}" REDIS_URL: "{{ .Values.redis_url }}"
REDIS_CRAWLS_DONE_KEY: "crawls-done" REDIS_CRAWLS_DONE_KEY: "crawls-done"

2
chart/templates/frontend.yaml
View File

@ -10,7 +10,7 @@ spec:
matchLabels: matchLabels:
app: {{ .Values.name }} app: {{ .Values.name }}
role: frontend role: frontend
replicas: 1 replicas: {{ .Values.nginx_num_replicas | default 1 }}
template: template:
metadata: metadata:
labels: labels:

6
chart/values.yaml
View File

@ -45,6 +45,8 @@ api_limits_memory: "192Mi"
nginx_image: "nginx" nginx_image: "nginx"
nginx_pull_policy: "IfNotPresent" nginx_pull_policy: "IfNotPresent"
nginx_num_replicas: 1
nginx_requests_cpu: "3m" nginx_requests_cpu: "3m"
nginx_limits_cpu: "10m" nginx_limits_cpu: "10m"
@ -98,6 +100,10 @@ crawler_pull_policy: "IfNotPresent"
crawler_namespace: "crawlers" crawler_namespace: "crawlers"
# optional: enable to use a persist volume claim for all crawls
# can be enabled to use a multi-write shared filesystem
# crawler_pv_claim: "nfs-shared-crawls"
# num retries # num retries
crawl_retries: 3 crawl_retries: 3