From d9e73fcbc3122407e60a5e1c4af75c898bed7bbb Mon Sep 17 00:00:00 2001 From: Henry Wilkinson Date: Sat, 8 Jul 2023 11:54:30 -0400 Subject: [PATCH] Reorder Limits section (#966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Reorder Limits section - Minor text change to section names - "Limit Per Page" → "Per-Page Limits" - "Limit Per Crawl" → "Per-Crawl Limits" * Reorder limits section in documentation --- docs/user-guide/workflow-setup.md | 24 ++-- frontend/src/pages/org/workflow-editor.ts | 152 +++++++++++----------- 2 files changed, 88 insertions(+), 88 deletions(-) diff --git a/docs/user-guide/workflow-setup.md b/docs/user-guide/workflow-setup.md index e2c3c5a2..6317415b 100644 --- a/docs/user-guide/workflow-setup.md +++ b/docs/user-guide/workflow-setup.md @@ -98,6 +98,18 @@ This can be useful for avoiding crawler traps — sites that may automatically g ## Limits +### Max Pages + +Adds a hard limit on the number of pages that will be crawled. The crawl will be gracefully stopped after this limit is reached. + +### Crawl Time Limit + +The crawl will be gracefully stopped after this set period of time. + +### Crawler Instances + +Increasing the amount of crawler instances will speed up crawls by using additional browser windows to capture more pages in parallel. This will also increase the amount of traffic sent to the website and may result in a higher chance of getting rate limited. + ### Page Load Timeout Limits amount of time to wait for a page to load. Behaviors will run after this timeout only if the page is partially or fully loaded. @@ -114,18 +126,6 @@ When enabled, the browser will automatically scroll to the end of the page. Waits on the page for a set period of time after any behaviors have finished running. This can be helpful to avoid rate limiting however it will slow down your crawl. -### Max Pages - -Adds a hard limit on the number of pages that will be crawled. The crawl will be gracefully stopped after this limit is reached. - -### Crawl Time Limit - -The crawl will be gracefully stopped after this set period of time. - -### Crawler Instances - -Increasing the amount of crawler instances will speed up crawls by using additional browser windows to capture more pages in parallel. This will also increase the amount of traffic sent to the website and may result in a higher chance of getting rate limited. - ## Browser Settings ### Browser Profile diff --git a/frontend/src/pages/org/workflow-editor.ts b/frontend/src/pages/org/workflow-editor.ts index ebcbc391..c780fd8b 100644 --- a/frontend/src/pages/org/workflow-editor.ts +++ b/frontend/src/pages/org/workflow-editor.ts @@ -1236,82 +1236,7 @@ https://archiveweb.page/images/${"logo.svg"}`} inputEl.helpText = helpText; }; return html` - ${this.renderSectionHeading(msg("Limit Per Page"))} - ${this.renderFormCol(html` - - ${msg("seconds")} - - `)} - ${this.renderHelpTextCol( - msg( - `Limits amount of time to wait for a page to load. Behaviors will run after this timeout only if the page is partially or fully loaded.` - ) - )} - ${this.renderFormCol(html` - - ${msg("seconds")} - - `)} - ${this.renderHelpTextCol( - msg(`Limits how long behaviors can run on each page.`) - )} - ${this.renderFormCol(html` - ${msg("Auto-Scroll Behavior")} - `)} - ${this.renderHelpTextCol( - msg( - `When enabled the browser will automatically scroll to the end of the page.` - ), - false - )} - ${this.renderFormCol(html` - - ${msg("seconds")} - - `)} - ${this.renderHelpTextCol( - msg( - `Waits on the page after behaviors are complete before moving onto the next page. Can be helpful for rate limiting.` - ) - )} - ${this.renderSectionHeading(msg("Limit Per Crawl"))} + ${this.renderSectionHeading(msg("Per-Crawl Limits"))} ${this.renderFormCol(html` + ${msg("seconds")} + + `)} + ${this.renderHelpTextCol( + msg( + `Limits amount of time to wait for a page to load. Behaviors will run after this timeout only if the page is partially or fully loaded.` + ) + )} + ${this.renderFormCol(html` + + ${msg("seconds")} + + `)} + ${this.renderHelpTextCol( + msg(`Limits how long behaviors can run on each page.`) + )} + ${this.renderFormCol(html` + ${msg("Auto-Scroll Behavior")} + `)} + ${this.renderHelpTextCol( + msg( + `When enabled the browser will automatically scroll to the end of the page.` + ), + false + )} + ${this.renderFormCol(html` + + ${msg("seconds")} + + `)} + ${this.renderHelpTextCol( + msg( + `Waits on the page after behaviors are complete before moving onto the next page. Can be helpful for rate limiting.` + ) + )} `; }