tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Nightly Tests Fix (#905)

Browse Source 
* tests: fix nightly test to account for 'waiting_capacity' state

* readd missing --logErrorsToRedis flag
This commit is contained in:
Ilya Kreymer 2023-06-02 21:47:41 -07:00 committed by GitHub
parent 3f42515914
commit f2b7b6bcd5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/test_nightly/conftest.py
View File

@ -231,11 +231,10 @@ def timeout_crawl(admin_auth_headers, default_org_id):
def error_crawl_id(admin_auth_headers, default_org_id):
crawl_data = {
"runNow": True,
"name": "Youtube crawl with errors",
"name": "Invalid URL crawl",
"config": {
"seeds": [
{"url": "https://invalid.webrecorder.net/"},
{"url": "https://invalid-2.webrecorder.net/"},
{"url": "https://invalid-x.webrecorder.net/"},
],
"limit": 1,
},

2
backend/test_nightly/test_concurrent_crawl_limit.py
View File

@ -26,7 +26,7 @@ def test_run_two_only_one_concurrent(org_with_quotas, admin_auth_headers):
crawl_id_b = run_crawl(org_with_quotas, admin_auth_headers)
while (
get_crawl_status(org_with_quotas, crawl_id_a, admin_auth_headers) == "starting"
get_crawl_status(org_with_quotas, crawl_id_a, admin_auth_headers) in ("starting", "waiting_capacity")
):
time.sleep(2)

2
chart/templates/configmap.yaml
View File

@ -71,7 +71,7 @@ metadata:
namespace: {{ .Values.crawler_namespace }}
data:
CRAWL_ARGS: "--workers {{ .Values.crawler_browser_instances | default 1 }} --sizeLimit {{ .Values.crawler_session_size_limit_bytes }} --timeLimit {{ .Values.crawler_session_time_limit_seconds }} --maxPageLimit {{ .Values.max_pages_per_crawl | default 0 }} --healthCheckPort {{ .Values.crawler_liveness_port }} --diskUtilization {{ .Values.disk_utilization_threshold }} --logging {{ .Values.crawler_logging_opts }} --text {{ .Values.crawler_extract_full_text }} --generateWACZ --waitOnDone --collection thecrawl --screencastPort 9037"
CRAWL_ARGS: "--workers {{ .Values.crawler_browser_instances | default 1 }} --sizeLimit {{ .Values.crawler_session_size_limit_bytes }} --timeLimit {{ .Values.crawler_session_time_limit_seconds }} --maxPageLimit {{ .Values.max_pages_per_crawl | default 0 }} --healthCheckPort {{ .Values.crawler_liveness_port }} --diskUtilization {{ .Values.disk_utilization_threshold }} --logging {{ .Values.crawler_logging_opts }} --text {{ .Values.crawler_extract_full_text }} --generateWACZ --waitOnDone --collection thecrawl --screencastPort 9037 --logErrorsToRedis"
---
apiVersion: v1