From 18b2c1abfcbdea411dd794eeb22b4b9bbcaf78d6 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Tue, 26 Sep 2023 10:29:03 -0700 Subject: [PATCH] limit: set default page limit to 50k pages (#1211) --- chart/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chart/values.yaml b/chart/values.yaml index b849e823..86391a1f 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -22,8 +22,9 @@ crawler_extract_full_text: false # max pages per crawl # set to non-zero value to enforce global max pages per crawl limit +# if 0, there is no page limit (may need to adjust crawler/redis settings for larger crawls) # if set, each workflow can have a lower limit, but not higher -max_pages_per_crawl: 0 +max_pages_per_crawl: 50000 # User Agent Options # set to add suffix to default browser User Agent