crawl config new template: add support for 'extraHops' config option (available in browsertrix-crawler 0.5.0) (#104)

frontend:
- add checkbox to basic crawl config component which sets 'extraHops' to 1, otherwise to 0
- text tweaks: rename Scope Type -> Crawl Scope, capitalization

backend: add 'extraHops' to CrawlConfig
fixes #102
This commit is contained in:
Ilya Kreymer 2022-01-26 21:18:22 -08:00 committed by GitHub
parent 2666b6f6aa
commit 0bea0cfff2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 14 additions and 4 deletions

View File

@ -55,6 +55,7 @@ class RawCrawlConfig(BaseModel):
depth: Optional[int] = -1
limit: Optional[int] = 0
extraHops: Optional[int] = 0
behaviorTimeout: Optional[int] = 90

View File

@ -227,7 +227,7 @@ export class CrawlTemplatesNew extends LiteElement {
<div class="pr-2 flex-1">
<sl-select
name="schedule"
label=${msg("Recurring crawls")}
label=${msg("Recurring Crawls")}
value=${this.scheduleInterval}
@sl-select=${(e: any) =>
(this.scheduleInterval = e.target.value)}
@ -311,7 +311,7 @@ export class CrawlTemplatesNew extends LiteElement {
<sl-input
name="crawlTimeoutMinutes"
label=${msg("Time limit")}
label=${msg("Time Limit")}
placeholder=${msg("unlimited")}
type="number"
>
@ -365,7 +365,7 @@ export class CrawlTemplatesNew extends LiteElement {
></sl-textarea>
<sl-select
name="scopeType"
label=${msg("Scope type")}
label=${msg("Crawl Scope")}
value=${this.initialCrawlConfig!.scopeType!}
>
<sl-menu-item value="page">Page</sl-menu-item>
@ -374,9 +374,13 @@ export class CrawlTemplatesNew extends LiteElement {
<sl-menu-item value="host">Host</sl-menu-item>
<sl-menu-item value="any">Any</sl-menu-item>
</sl-select>
<sl-checkbox
name="extraHopsOne"
>${msg("Include External Links ('one hop out')")}
</sl-checkbox>
<sl-input
name="limit"
label=${msg("Page limit")}
label=${msg("Page Limit")}
type="number"
value=${ifDefined(this.initialCrawlConfig!.limit)}
placeholder=${msg("unlimited")}
@ -497,6 +501,7 @@ export class CrawlTemplatesNew extends LiteElement {
.map((url) => ({ url })),
scopeType: formData.get("scopeType") as string,
limit: pageLimit ? +pageLimit : 0,
extraHops: formData.get("extraHopsOne") ? 1 : 0,
};
}

View File

@ -1,6 +1,7 @@
type SeedConfig = {
scopeType?: string;
limit?: number;
extraHops?: number;
};
export type CrawlConfig = {

View File

@ -8,6 +8,9 @@ import "@shoelace-style/shoelace/dist/components/alert/alert";
import(
/* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/button/button"
);
import(
/* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/checkbox/checkbox"
);
import(
/* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/details/details"
);