diff --git a/frontend/src/components/config-details.ts b/frontend/src/components/config-details.ts index 153b6c58..466b4917 100644 --- a/frontend/src/components/config-details.ts +++ b/frontend/src/components/config-details.ts @@ -88,6 +88,12 @@ export class ConfigDetails extends LiteElement { `, () => this.renderSetting(msg("Exclusions"), msg("None")) )} + ${this.renderSetting( + msg("Page Time Limit"), + crawlConfig?.config.behaviorTimeout + ? msg(str`${crawlConfig?.config.behaviorTimeout / 60} minute(s)`) + : msg("None") + )} ${this.renderSetting( msg("Crawl Time Limit"), crawlConfig?.crawlTimeout @@ -111,9 +117,9 @@ export class ConfigDetails extends LiteElement { crawlConfig?.profileid, () => html` ${crawlConfig?.profileName} @@ -129,12 +135,6 @@ export class ConfigDetails extends LiteElement { msg("Language"), ISO6391.getName(crawlConfig?.config.lang!) )} - ${this.renderSetting( - msg("Page Time Limit"), - crawlConfig?.config.behaviorTimeout - ? msg(str`${crawlConfig?.config.behaviorTimeout / 60} minute(s)`) - : msg("None") - )}
diff --git a/frontend/src/pages/org/crawl-config-editor.ts b/frontend/src/pages/org/crawl-config-editor.ts index 5e019c7a..08ebdaa4 100644 --- a/frontend/src/pages/org/crawl-config-editor.ts +++ b/frontend/src/pages/org/crawl-config-editor.ts @@ -176,6 +176,7 @@ function validURL(url: string) { const trimExclusions = flow(uniq, compact); const urlListToArray = (str: string) => str.trim().replace(/,/g, " ").split(/\s+/g); +const DEFAULT_BEHAVIOR_TIMEOUT_MINUTES = 5; @localized() export class CrawlConfigEditor extends LiteElement { @@ -200,6 +201,9 @@ export class CrawlConfigEditor extends LiteElement { @state() private progressState!: ProgressState; + @state() + private defaultBehaviorTimeoutMinutes?: number; + @state() private formState!: FormState; @@ -274,6 +278,9 @@ export class CrawlConfigEditor extends LiteElement { } willUpdate(changedProperties: Map) { + if (changedProperties.has("authState") && this.authState) { + this.fetchAPIDefaults(); + } if ( changedProperties.get("initialCrawlConfig") && this.initialCrawlConfig @@ -393,9 +400,13 @@ export class CrawlConfigEditor extends LiteElement { if (this.initialCrawlConfig.tags?.length) { formState.tags = this.initialCrawlConfig.tags; } - if (this.initialCrawlConfig.crawlTimeout) { + if (typeof this.initialCrawlConfig.crawlTimeout === "number") { formState.crawlTimeoutMinutes = this.initialCrawlConfig.crawlTimeout / 60; } + if (typeof this.initialCrawlConfig.config.behaviorTimeout === "number") { + formState.pageTimeoutMinutes = + this.initialCrawlConfig.config.behaviorTimeout / 60; + } return { jobName: this.initialCrawlConfig.name, @@ -1016,6 +1027,26 @@ https://example.net`} private renderCrawlScale() { return html` ${this.renderSectionHeading(msg("Crawl Limits"))} + ${this.renderFormCol(html` + + ${msg("minutes")} + + `)} + ${this.renderHelpTextCol( + html`Adds a hard time limit for how long the crawler can spend on a + single webpage.` + )} ${this.renderFormCol(html` - ${msg("minutes")} - - `)} - ${this.renderHelpTextCol( - html`Adds a hard time limit for how long the crawler can spend on a - single webpage.` - )} `; } @@ -1638,9 +1653,7 @@ https://example.net`} if (crawlId) { this.navTo(`/orgs/${this.orgId}/crawls/crawl/${crawlId}`); } else { - this.navTo( - `/orgs/${this.orgId}/crawl-templates/config/${data.added}` - ); + this.navTo(`/orgs/${this.orgId}/crawl-templates/config/${data.added}`); } } catch (e: any) { if (e?.isApiError) { @@ -1707,9 +1720,10 @@ https://example.net`} ...(this.jobType === "seed-crawl" ? this.parseSeededConfig() : this.parseUrlListConfig()), - behaviorTimeout: this.formState.pageTimeoutMinutes - ? this.formState.pageTimeoutMinutes * 60 - : 0, + behaviorTimeout: + (this.formState.pageTimeoutMinutes ?? + this.defaultBehaviorTimeoutMinutes ?? + DEFAULT_BEHAVIOR_TIMEOUT_MINUTES) * 60, limit: this.formState.pageLimit ? +this.formState.pageLimit : null, extraHops: this.formState.includeLinkedPages ? 1 : 0, lang: this.formState.lang || null, @@ -1806,6 +1820,21 @@ https://example.net`} this.formState = mergeDeep(this.formState, nextState); } } + + private async fetchAPIDefaults() { + try { + const data = await this.apiFetch("/settings", this.authState!); + if (data.defaultBehaviorTimeSeconds) { + this.defaultBehaviorTimeoutMinutes = + data.defaultBehaviorTimeSeconds / 60; + } else { + this.defaultBehaviorTimeoutMinutes = DEFAULT_BEHAVIOR_TIMEOUT_MINUTES; + } + } catch (e: any) { + console.debug(e); + this.defaultBehaviorTimeoutMinutes = DEFAULT_BEHAVIOR_TIMEOUT_MINUTES; + } + } } customElements.define("btrix-crawl-config-editor", CrawlConfigEditor); diff --git a/frontend/src/pages/org/types.ts b/frontend/src/pages/org/types.ts index 9088cc29..188ac988 100644 --- a/frontend/src/pages/org/types.ts +++ b/frontend/src/pages/org/types.ts @@ -71,7 +71,7 @@ export type InitialCrawlConfig = Pick< jobType?: JobType; config: Pick< CrawlConfigParams["config"], - "seeds" | "scopeType" | "exclude" + "seeds" | "scopeType" | "exclude" | "behaviorTimeout" > & { extraHops?: CrawlConfigParams["config"]["extraHops"]; };