From 0bea0cfff20344a373bf0ef9b1323fb2767dde3c Mon Sep 17 00:00:00 2001
From: Ilya Kreymer <ikreymer@users.noreply.github.com>
Date: Wed, 26 Jan 2022 21:18:22 -0800
Subject: [PATCH] crawl config new template: add support for 'extraHops' config
 option (available in browsertrix-crawler 0.5.0) (#104)

frontend:
- add checkbox to basic crawl config component which sets 'extraHops' to 1, otherwise to 0
- text tweaks: rename Scope Type -> Crawl Scope, capitalization

backend: add 'extraHops' to CrawlConfig
fixes #102
---
 backend/crawlconfigs.py                           |  1 +
 frontend/src/pages/archive/crawl-templates-new.ts | 13 +++++++++----
 frontend/src/pages/archive/types.ts               |  1 +
 frontend/src/shoelace.ts                          |  3 +++
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/backend/crawlconfigs.py b/backend/crawlconfigs.py
index acb6b92b..bd4e568d 100644
--- a/backend/crawlconfigs.py
+++ b/backend/crawlconfigs.py
@@ -55,6 +55,7 @@ class RawCrawlConfig(BaseModel):
 
     depth: Optional[int] = -1
     limit: Optional[int] = 0
+    extraHops: Optional[int] = 0
 
     behaviorTimeout: Optional[int] = 90
 
diff --git a/frontend/src/pages/archive/crawl-templates-new.ts b/frontend/src/pages/archive/crawl-templates-new.ts
index 0e679ca7..e0c1a2cf 100644
--- a/frontend/src/pages/archive/crawl-templates-new.ts
+++ b/frontend/src/pages/archive/crawl-templates-new.ts
@@ -227,7 +227,7 @@ export class CrawlTemplatesNew extends LiteElement {
             <div class="pr-2 flex-1">
               <sl-select
                 name="schedule"
-                label=${msg("Recurring crawls")}
+                label=${msg("Recurring Crawls")}
                 value=${this.scheduleInterval}
                 @sl-select=${(e: any) =>
                   (this.scheduleInterval = e.target.value)}
@@ -311,7 +311,7 @@ export class CrawlTemplatesNew extends LiteElement {
 
         <sl-input
           name="crawlTimeoutMinutes"
-          label=${msg("Time limit")}
+          label=${msg("Time Limit")}
           placeholder=${msg("unlimited")}
           type="number"
         >
@@ -365,7 +365,7 @@ export class CrawlTemplatesNew extends LiteElement {
       ></sl-textarea>
       <sl-select
         name="scopeType"
-        label=${msg("Scope type")}
+        label=${msg("Crawl Scope")}
         value=${this.initialCrawlConfig!.scopeType!}
       >
         <sl-menu-item value="page">Page</sl-menu-item>
@@ -374,9 +374,13 @@ export class CrawlTemplatesNew extends LiteElement {
         <sl-menu-item value="host">Host</sl-menu-item>
         <sl-menu-item value="any">Any</sl-menu-item>
       </sl-select>
+      <sl-checkbox
+        name="extraHopsOne"
+      >${msg("Include External Links ('one hop out')")}
+      </sl-checkbox>
       <sl-input
         name="limit"
-        label=${msg("Page limit")}
+        label=${msg("Page Limit")}
         type="number"
         value=${ifDefined(this.initialCrawlConfig!.limit)}
         placeholder=${msg("unlimited")}
@@ -497,6 +501,7 @@ export class CrawlTemplatesNew extends LiteElement {
           .map((url) => ({ url })),
         scopeType: formData.get("scopeType") as string,
         limit: pageLimit ? +pageLimit : 0,
+        extraHops: formData.get("extraHopsOne") ? 1 : 0,
       };
     }
 
diff --git a/frontend/src/pages/archive/types.ts b/frontend/src/pages/archive/types.ts
index 9481f103..ebe1a0ce 100644
--- a/frontend/src/pages/archive/types.ts
+++ b/frontend/src/pages/archive/types.ts
@@ -1,6 +1,7 @@
 type SeedConfig = {
   scopeType?: string;
   limit?: number;
+  extraHops?: number;
 };
 
 export type CrawlConfig = {
diff --git a/frontend/src/shoelace.ts b/frontend/src/shoelace.ts
index 07362029..39cc64c7 100644
--- a/frontend/src/shoelace.ts
+++ b/frontend/src/shoelace.ts
@@ -8,6 +8,9 @@ import "@shoelace-style/shoelace/dist/components/alert/alert";
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/button/button"
 );
+import(
+  /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/checkbox/checkbox"
+);
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/details/details"
 );