From b2088f563414e0a8b8665251cbadf7c1d24f6e78 Mon Sep 17 00:00:00 2001
From: sua yoo <sua@suayoo.com>
Date: Sun, 16 Jan 2022 14:43:33 -0800
Subject: [PATCH] Add initial crawl template form (#80)

---
 frontend/src/components/input/input.css       |   2 +-
 frontend/src/components/input/input.ts        |   4 +
 frontend/src/index.ts                         |   8 +-
 frontend/src/pages/archive/crawl-templates.ts | 240 ++++++++++++++++++
 .../pages/{archive.ts => archive/index.ts}    |  78 +++++-
 frontend/src/routes.ts                        |   1 +
 frontend/src/shoelace.ts                      |   6 +
 frontend/src/utils/APIRouter.ts               |   4 +-
 frontend/src/utils/LiteElement.ts             |   4 +-
 9 files changed, 328 insertions(+), 19 deletions(-)
 create mode 100644 frontend/src/pages/archive/crawl-templates.ts
 rename frontend/src/pages/{archive.ts => archive/index.ts} (71%)

diff --git a/frontend/src/components/input/input.css b/frontend/src/components/input/input.css
index c58ff03d..6f61265f 100644
--- a/frontend/src/components/input/input.css
+++ b/frontend/src/components/input/input.css
@@ -23,7 +23,7 @@
   border: solid var(--sl-input-border-width) var(--sl-input-border-color);
   border-radius: var(--sl-input-border-radius-medium);
   box-shadow: none;
-  padding: var(--sl-input-spacing-medium);
+  padding: 0 var(--sl-input-spacing-medium);
   cursor: inherit;
   -webkit-appearance: none;
   transition: var(--sl-transition-fast) border,
diff --git a/frontend/src/components/input/input.ts b/frontend/src/components/input/input.ts
index ba5d59ba..bf2e95a9 100644
--- a/frontend/src/components/input/input.ts
+++ b/frontend/src/components/input/input.ts
@@ -33,6 +33,9 @@ export class Input extends LiteElement {
   @property({ type: String })
   placeholder?: string;
 
+  @property()
+  value?: any;
+
   @property()
   autocomplete?: any;
 
@@ -60,6 +63,7 @@ export class Input extends LiteElement {
             : ifDefined(this.type as any)}
           autocomplete=${ifDefined(this.autocomplete)}
           placeholder=${ifDefined(this.placeholder)}
+          value=${ifDefined(this.value)}
           ?required=${Boolean(this.required)}
         />
         ${this.togglePassword
diff --git a/frontend/src/index.ts b/frontend/src/index.ts
index 98c72f4d..2d91f664 100644
--- a/frontend/src/index.ts
+++ b/frontend/src/index.ts
@@ -197,7 +197,7 @@ export class App extends LiteElement {
     return html`
       <div class="bg-gray-900 text-gray-50">
         <nav
-          class="max-w-screen-xl mx-auto p-2 box-border flex items-center justify-between"
+          class="max-w-screen-lg mx-auto p-2 box-border flex items-center justify-between"
         >
           <div>
             <a href="/archives" @click="${this.navLink}"
@@ -268,7 +268,7 @@ export class App extends LiteElement {
       </li>
     `;
     const appLayout = (template: TemplateResult) => html`
-      <div class="w-full max-w-screen-xl mx-auto p-2 md:py-8 box-border">
+      <div class="w-full max-w-screen-lg mx-auto p-2 md:py-8 box-border">
         ${template}
       </div>
     `;
@@ -367,6 +367,7 @@ export class App extends LiteElement {
 
       case "archive":
       case "archiveAddMember":
+      case "archiveNewResourceTab":
         return appLayout(html`<btrix-archive
           class="w-full"
           @navigate=${this.onNavigateTo}
@@ -376,6 +377,7 @@ export class App extends LiteElement {
           archiveId=${this.viewState.params.id}
           archiveTab=${this.viewState.params.tab as ArchiveTab}
           ?isAddingMember=${this.viewState.route === "archiveAddMember"}
+          ?isNewResourceTab=${this.viewState.route === "archiveNewResourceTab"}
         ></btrix-archive>`);
 
       case "accountSettings":
@@ -462,6 +464,8 @@ export class App extends LiteElement {
   }
 
   onNavigateTo(event: NavigateEvent) {
+    event.stopPropagation();
+
     this.navigate(event.detail);
   }
 
diff --git a/frontend/src/pages/archive/crawl-templates.ts b/frontend/src/pages/archive/crawl-templates.ts
new file mode 100644
index 00000000..7efd1c2d
--- /dev/null
+++ b/frontend/src/pages/archive/crawl-templates.ts
@@ -0,0 +1,240 @@
+import { state, property } from "lit/decorators.js";
+import { msg, localized } from "@lit/localize";
+
+import type { AuthState } from "../../utils/AuthService";
+import LiteElement, { html } from "../../utils/LiteElement";
+
+type CrawlTemplate = any; // TODO
+
+const initialValues = {
+  name: `Example crawl ${Date.now()}`, // TODO remove placeholder
+  runNow: true,
+  schedule: "@weekly",
+  // crawlTimeoutMinutes: 0,
+  seedUrls: "",
+  scopeType: "prefix",
+  // limit: 0,
+};
+
+@localized()
+export class CrawlTemplates extends LiteElement {
+  @property({ type: Object })
+  authState!: AuthState;
+
+  @property({ type: String })
+  archiveId!: string;
+
+  @property({ type: Boolean })
+  isNew!: boolean;
+
+  @property({ type: Array })
+  crawlTemplates?: CrawlTemplate[];
+
+  @state()
+  isRunNow: boolean = initialValues.runNow;
+
+  render() {
+    if (this.isNew) {
+      return this.renderNew();
+    }
+
+    return this.renderList();
+  }
+
+  private renderNew() {
+    return html`
+      <h2 class="text-xl font-bold">${msg("New Crawl Template")}</h2>
+      <p>
+        ${msg(
+          "Configure a new crawl template. You can choose to run a crawl immediately upon saving this template."
+        )}
+      </p>
+
+      <main class="mt-4">
+        <sl-form @sl-submit=${this.onSubmit}>
+          <div class="border rounded-lg md:grid grid-cols-4">
+            <div class="col-span-1 p-4 md:p-8 md:border-b">
+              <h3 class="text-lg font-medium">${msg("Basic settings")}</h3>
+            </div>
+            <section class="col-span-3 p-4 md:p-8 border-b grid gap-5">
+              <div>
+                <sl-input
+                  name="name"
+                  label=${msg("Name")}
+                  placeholder=${msg("Example (example.com) Weekly Crawl", {
+                    desc: "Example crawl template name",
+                  })}
+                  autocomplete="off"
+                  value=${initialValues.name}
+                  required
+                ></sl-input>
+              </div>
+              <div class="flex items-end">
+                <!-- TODO schedule time -->
+                <div>
+                  <sl-select
+                    name="schedule"
+                    label=${msg("Schedule")}
+                    value=${initialValues.schedule}
+                  >
+                    <sl-menu-item value="">None</sl-menu-item>
+                    <sl-menu-item value="@daily">Daily</sl-menu-item>
+                    <sl-menu-item value="@weekly">Weekly</sl-menu-item>
+                    <sl-menu-item value="@monthly">Monthly</sl-menu-item>
+                  </sl-select>
+                </div>
+                <!-- <div>
+                  <btrix-input
+                    name="scheduleTime"
+                    type="time"
+                  ></btrix-input>
+                </div> -->
+              </div>
+              <div>
+                <sl-switch
+                  name="runNow"
+                  ?checked=${initialValues.runNow}
+                  @sl-change=${(e: any) => (this.isRunNow = e.target.checked)}
+                  >${msg("Run immediately")}</sl-switch
+                >
+              </div>
+
+              <div>
+                <sl-input
+                  name="crawlTimeoutMinutes"
+                  label=${msg("Time limit")}
+                  placeholder=${msg("unlimited")}
+                  type="number"
+                >
+                  <span slot="suffix">${msg("minutes")}</span>
+                </sl-input>
+              </div>
+            </section>
+
+            <div class="col-span-1 p-4 md:p-8 md:border-b">
+              <h3 class="text-lg font-medium">${msg("Pages")}</h3>
+            </div>
+            <section class="col-span-3 p-4 md:p-8 border-b grid gap-5">
+              <div>
+                <sl-textarea
+                  name="seedUrls"
+                  label=${msg("Seed URLs")}
+                  helpText=${msg("Separated by a new line, space or comma")}
+                  placeholder=${msg(
+                    `https://webrecorder.net\nhttps://example.com`,
+                    {
+                      desc: "Example seed URLs",
+                    }
+                  )}
+                  help-text=${msg(
+                    "Separate URLs with a new line, space or comma."
+                  )}
+                  rows="3"
+                  value=${initialValues.seedUrls}
+                  required
+                ></sl-textarea>
+              </div>
+              <div>
+                <sl-select
+                  name="scopeType"
+                  label=${msg("Scope type")}
+                  value=${initialValues.scopeType}
+                >
+                  <sl-menu-item value="page">Page</sl-menu-item>
+                  <sl-menu-item value="page-spa">Page SPA</sl-menu-item>
+                  <sl-menu-item value="prefix">Prefix</sl-menu-item>
+                  <sl-menu-item value="host">Host</sl-menu-item>
+                  <sl-menu-item value="any">Any</sl-menu-item>
+                </sl-select>
+              </div>
+              <div>
+                <sl-input
+                  name="limit"
+                  label=${msg("Page limit")}
+                  type="number"
+                  placeholder=${msg("unlimited")}
+                >
+                  <span slot="suffix">${msg("pages")}</span>
+                </sl-input>
+              </div>
+            </section>
+
+            <div class="col-span-4 p-4 md:p-8 text-center">
+              ${this.isRunNow
+                ? html`
+                    <p class="text-sm mb-3">
+                      ${msg("A crawl will start immediately on save.")}
+                    </p>
+                  `
+                : ""}
+
+              <sl-button type="primary" submit
+                >${msg("Save Crawl Template")}</sl-button
+              >
+            </div>
+          </div>
+        </sl-form>
+      </main>
+    `;
+  }
+
+  private renderList() {
+    return html`
+      <div class="text-center">
+        <sl-button
+          @click=${() =>
+            this.navTo(`/archives/${this.archiveId}/crawl-templates/new`)}
+        >
+          <sl-icon slot="prefix" name="plus-square-dotted"></sl-icon>
+          ${msg("Create new crawl template")}
+        </sl-button>
+      </div>
+
+      <div>
+        ${this.crawlTemplates?.map(
+          (template) => html`<div>${template.id}</div>`
+        )}
+      </div>
+    `;
+  }
+
+  private async onSubmit(event: { detail: { formData: FormData } }) {
+    if (!this.authState) return;
+
+    const { formData } = event.detail;
+
+    const crawlTimeoutMinutes = formData.get("crawlTimeoutMinutes");
+    const pageLimit = formData.get("limit");
+    const seedUrlsStr = formData.get("seedUrls");
+    const params = {
+      name: formData.get("name"),
+      schedule: formData.get("schedule"),
+      runNow: this.isRunNow,
+      crawlTimeout: crawlTimeoutMinutes ? +crawlTimeoutMinutes * 60 : 0,
+      config: {
+        seeds: (seedUrlsStr as string).trim().replace(/,/g, " ").split(/\s+/g),
+        scopeType: formData.get("scopeType"),
+        limit: pageLimit ? +pageLimit : 0,
+      },
+    };
+
+    console.log(params);
+
+    try {
+      await this.apiFetch(
+        `/archives/${this.archiveId}/crawlconfigs/`,
+        this.authState,
+        {
+          method: "POST",
+          body: JSON.stringify(params),
+        }
+      );
+
+      console.debug("success");
+
+      this.navTo(`/archives/${this.archiveId}/crawl-templates`);
+    } catch (e) {
+      console.error(e);
+    }
+  }
+}
diff --git a/frontend/src/pages/archive.ts b/frontend/src/pages/archive/index.ts
similarity index 71%
rename from frontend/src/pages/archive.ts
rename to frontend/src/pages/archive/index.ts
index 9a865ea0..6d8b4720 100644
--- a/frontend/src/pages/archive.ts
+++ b/frontend/src/pages/archive/index.ts
@@ -1,14 +1,18 @@
 import { state, property } from "lit/decorators.js";
 import { msg, localized, str } from "@lit/localize";
 
-import type { AuthState } from "../utils/AuthService";
-import type { CurrentUser } from "../types/user";
-import type { ArchiveData } from "../utils/archives";
-import LiteElement, { html } from "../utils/LiteElement";
-import { needLogin } from "../utils/auth";
-import { isOwner } from "../utils/archives";
+import type { AuthState } from "../../utils/AuthService";
+import type { CurrentUser } from "../../types/user";
+import type { ArchiveData } from "../../utils/archives";
+import LiteElement, { html } from "../../utils/LiteElement";
+import { needLogin } from "../../utils/auth";
+import { isOwner } from "../../utils/archives";
+import { CrawlTemplates } from "./crawl-templates";
 
-export type ArchiveTab = "settings" | "members";
+customElements.define("btrix-crawl-templates", CrawlTemplates);
+
+export type ArchiveTab = "crawl-templates" | "settings" | "members";
+type CrawlTemplate = any; // TODO
 
 const defaultTab = "settings";
 
@@ -30,9 +34,16 @@ export class Archive extends LiteElement {
   @property({ type: Boolean })
   isAddingMember: boolean = false;
 
+  /** Whether new resource is being added in tab */
+  @property({ type: Boolean })
+  isNewResourceTab: boolean = false;
+
   @state()
   private archive?: ArchiveData;
 
+  @state()
+  private crawlTemplates?: CrawlTemplate[];
+
   @state()
   private successfullyInvitedEmail?: string;
 
@@ -50,8 +61,18 @@ export class Archive extends LiteElement {
     }
   }
 
-  updated(changedProperties: any) {
-    if (changedProperties.has("isAddingMember") && this.isAddingMember) {
+  async updated(changedProperties: any) {
+    if (
+      changedProperties.has("archiveTab") &&
+      this.archiveTab === "crawl-templates" &&
+      !this.isNewResourceTab
+    ) {
+      this.crawlTemplates = await this.getCrawlTemplates();
+
+      if (!this.crawlTemplates.length) {
+        this.navTo(`/archives/${this.archiveId}/crawl-templates/new`);
+      }
+    } else if (changedProperties.has("isAddingMember") && this.isAddingMember) {
       this.successfullyInvitedEmail = undefined;
     }
   }
@@ -65,7 +86,7 @@ export class Archive extends LiteElement {
       </div>`;
     }
 
-    const showMembers = Boolean(this.archive.users);
+    const showMembersTab = Boolean(this.archive.users);
 
     return html`<article class="grid gap-4">
       <nav class="font-medium text-sm text-gray-500">
@@ -81,13 +102,19 @@ export class Archive extends LiteElement {
 
       <main>
         <sl-tab-group @sl-tab-show=${this.updateUrl}>
+          <sl-tab
+            slot="nav"
+            panel="crawl-templates"
+            ?active=${this.archiveTab === "crawl-templates"}
+            >${msg("Crawl Templates")}</sl-tab
+          >
           <sl-tab
             slot="nav"
             panel="settings"
             ?active=${this.archiveTab === "settings"}
             >${msg("Settings")}</sl-tab
           >
-          ${showMembers
+          ${showMembersTab
             ? html`<sl-tab
                 slot="nav"
                 panel="members"
@@ -101,7 +128,12 @@ export class Archive extends LiteElement {
             ?active=${this.archiveTab === "settings"}
             >${this.renderSettings()}</sl-tab-panel
           >
-          ${showMembers
+          <sl-tab-panel
+            name="crawl-templates"
+            ?active=${this.archiveTab === "crawl-templates"}
+            >${this.renderCrawlTemplates()}</sl-tab-panel
+          >
+          ${showMembersTab
             ? html`<sl-tab-panel
                 name="members"
                 ?active=${this.archiveTab === "members"}
@@ -120,6 +152,19 @@ export class Archive extends LiteElement {
     return html` TODO `;
   }
 
+  private renderCrawlTemplates() {
+    if (!this.isNewResourceTab && !this.crawlTemplates) {
+      return html` TODO `;
+    }
+
+    return html`<btrix-crawl-templates
+      .authState=${this.authState!}
+      .archiveId=${this.archiveId!}
+      .crawlTemplates=${this.crawlTemplates}
+      .isNew=${this.isNewResourceTab}
+    ></btrix-crawl-templates>`;
+  }
+
   private renderMembers() {
     if (!this.archive!.users) return;
 
@@ -205,6 +250,15 @@ export class Archive extends LiteElement {
     return data;
   }
 
+  async getCrawlTemplates(): Promise<CrawlTemplate[]> {
+    const data = await this.apiFetch(
+      `/archives/${this.archiveId}/crawlconfigs`,
+      this.authState!
+    );
+
+    return data.crawl_configs;
+  }
+
   onInviteSuccess(
     event: CustomEvent<{ inviteEmail: string; isExistingUser: boolean }>
   ) {
diff --git a/frontend/src/routes.ts b/frontend/src/routes.ts
index 44292bae..75cd2945 100644
--- a/frontend/src/routes.ts
+++ b/frontend/src/routes.ts
@@ -12,6 +12,7 @@ export const ROUTES = {
   accountSettings: "/account/settings",
   archives: "/archives",
   archive: "/archives/:id/:tab",
+  archiveNewResourceTab: "/archives/:id/:tab/new",
   archiveAddMember: "/archives/:id/:tab/add-member",
   users: "/users",
   usersInvite: "/users/invite",
diff --git a/frontend/src/shoelace.ts b/frontend/src/shoelace.ts
index 218ba474..b7a87328 100644
--- a/frontend/src/shoelace.ts
+++ b/frontend/src/shoelace.ts
@@ -38,6 +38,9 @@ import(
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/spinner/spinner"
 );
+import(
+  /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/switch/switch"
+);
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/tab/tab"
 );
@@ -47,5 +50,8 @@ import(
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/tab-panel/tab-panel"
 );
+import(
+  /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/textarea/textarea"
+);
 
 setBasePath("/shoelace");
diff --git a/frontend/src/utils/APIRouter.ts b/frontend/src/utils/APIRouter.ts
index 57a3bcd1..e8e047fd 100644
--- a/frontend/src/utils/APIRouter.ts
+++ b/frontend/src/utils/APIRouter.ts
@@ -15,9 +15,7 @@ export type ViewState = {
   // e.g. "/redirect?url"
   params: { [key: string]: string };
 };
-export type NavigateEvent = {
-  detail: string;
-};
+export interface NavigateEvent extends CustomEvent {}
 
 export default class APIRouter {
   routes: Routes;
diff --git a/frontend/src/utils/LiteElement.ts b/frontend/src/utils/LiteElement.ts
index cf98e85f..10f426b9 100644
--- a/frontend/src/utils/LiteElement.ts
+++ b/frontend/src/utils/LiteElement.ts
@@ -11,7 +11,9 @@ export default class LiteElement extends LitElement {
   }
 
   navTo(url: string) {
-    this.dispatchEvent(new CustomEvent("navigate", { detail: url }));
+    this.dispatchEvent(
+      new CustomEvent("navigate", { detail: url, bubbles: true })
+    );
   }
 
   navLink(event: Event) {