crawl config new template: add support for 'extraHops' config option (available in browsertrix-crawler 0.5.0) (#104)

frontend:
- add checkbox to basic crawl config component which sets 'extraHops' to 1, otherwise to 0
- text tweaks: rename Scope Type -> Crawl Scope, capitalization

backend: add 'extraHops' to CrawlConfig
fixes #102

backend/crawlconfigs.py

@@ -55,6 +55,7 @@ class RawCrawlConfig(BaseModel):
 
     depth: Optional[int] = -1
     limit: Optional[int] = 0
+    extraHops: Optional[int] = 0
 
     behaviorTimeout: Optional[int] = 90
 

frontend/src/pages/archive/crawl-templates-new.ts

@@ -227,7 +227,7 @@ export class CrawlTemplatesNew extends LiteElement {
             <div class="pr-2 flex-1">
               <sl-select
                 name="schedule"
-                label=${msg("Recurring crawls")}
+                label=${msg("Recurring Crawls")}
                 value=${this.scheduleInterval}
                 @sl-select=${(e: any) =>
                   (this.scheduleInterval = e.target.value)}
@@ -311,7 +311,7 @@ export class CrawlTemplatesNew extends LiteElement {
 
         <sl-input
           name="crawlTimeoutMinutes"
-          label=${msg("Time limit")}
+          label=${msg("Time Limit")}
           placeholder=${msg("unlimited")}
           type="number"
         >
@@ -365,7 +365,7 @@ export class CrawlTemplatesNew extends LiteElement {
       ></sl-textarea>
       <sl-select
         name="scopeType"
-        label=${msg("Scope type")}
+        label=${msg("Crawl Scope")}
         value=${this.initialCrawlConfig!.scopeType!}
       >
         <sl-menu-item value="page">Page</sl-menu-item>
@@ -374,9 +374,13 @@ export class CrawlTemplatesNew extends LiteElement {
         <sl-menu-item value="host">Host</sl-menu-item>
         <sl-menu-item value="any">Any</sl-menu-item>
       </sl-select>
+      <sl-checkbox
+        name="extraHopsOne"
+      >${msg("Include External Links ('one hop out')")}
+      </sl-checkbox>
       <sl-input
         name="limit"
-        label=${msg("Page limit")}
+        label=${msg("Page Limit")}
         type="number"
         value=${ifDefined(this.initialCrawlConfig!.limit)}
         placeholder=${msg("unlimited")}
@@ -497,6 +501,7 @@ export class CrawlTemplatesNew extends LiteElement {
           .map((url) => ({ url })),
         scopeType: formData.get("scopeType") as string,
         limit: pageLimit ? +pageLimit : 0,
+        extraHops: formData.get("extraHopsOne") ? 1 : 0,
       };
     }
 

frontend/src/pages/archive/types.ts

@@ -1,6 +1,7 @@
 type SeedConfig = {
   scopeType?: string;
   limit?: number;
+  extraHops?: number;
 };
 
 export type CrawlConfig = {

frontend/src/shoelace.ts

@@ -8,6 +8,9 @@ import "@shoelace-style/shoelace/dist/components/alert/alert";
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/button/button"
 );
+import(
+  /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/checkbox/checkbox"
+);
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/details/details"
 );