tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

QA analysis scale configurable in helm chart (#1843)

Browse Source 
- allow configuring QA run scale via 'qa_scale' setting in helm values
(overriding any setting on the qa crawljob)
- adds additional comments to browser instances helm values settings for clarity
- fixes #1842
This commit is contained in:
Ilya Kreymer 2024-05-30 12:59:21 -07:00 committed by GitHub
parent 251aef3ac1
commit d42de92d75
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/btrixcloud/operator/crawls.py
View File

@ -146,6 +146,9 @@ class CrawlOperator(BaseOperator):
qa_source_crawl_id=spec.get("qaSourceCrawlId"), qa_source_crawl_id=spec.get("qaSourceCrawlId"),
) )
if crawl.qa_source_crawl_id:
crawl.scale = int(params.get("qa_scale", 1))
# if finalizing, crawl is being deleted # if finalizing, crawl is being deleted
if data.finalizing: if data.finalizing:
if not status.finished: if not status.finished:

2
chart/templates/configmap.yaml
View File

@ -128,6 +128,8 @@ data:
profile_browser_workdir_size: "{{ .Values.profile_browser_workdir_size | default "4Gi" }}" profile_browser_workdir_size: "{{ .Values.profile_browser_workdir_size | default "4Gi" }}"
qa_scale: "{{ .Values.qa_scale | default 1 }}"
crawler_node_type: "{{ .Values.crawler_node_type }}" crawler_node_type: "{{ .Values.crawler_node_type }}"
redis_node_type: "{{ .Values.redis_node_type }}" redis_node_type: "{{ .Values.redis_node_type }}"

9
chart/values.yaml
View File

@ -232,9 +232,16 @@ crawler_cpu_base: 900m
# base memory per for 1 browser # base memory per for 1 browser
crawler_memory_base: 1024Mi crawler_memory_base: 1024Mi
# number of browsers per crawler instances # number of browser workers per crawler instances
crawler_browser_instances: 2 crawler_browser_instances: 2
# number of browser workers per crawler instances for QA runs
# defaults to 'crawler_browser_instances' if not set
# qa_browser_instances: 2
# fixed scale (number of crawler pods) for QA runs
qa_scale: 1
# this value is added to crawler_cpu_base, for each additional browser # this value is added to crawler_cpu_base, for each additional browser
# crawler_cpu = crawler_cpu_base + crawler_pu_per_extra_browser * (crawler_browser_instances - 1) # crawler_cpu = crawler_cpu_base + crawler_pu_per_extra_browser * (crawler_browser_instances - 1)
crawler_extra_cpu_per_browser: 600m crawler_extra_cpu_per_browser: 600m