Allow user to configure crawls with JSON (#86)

frontend/src/components/copy-button.ts

@@ -0,0 +1,48 @@
+import { LitElement, html } from "lit";
+import { property, state } from "lit/decorators.js";
+import { msg, localized } from "@lit/localize";
+
+/**
+ * Copy text to clipboard on click
+ *
+ * Usage example:
+ * ```ts
+ * <btrix-copy-button .value=${value} @on-copied=${console.log}></btrix-copy-button>
+ * ```
+ *
+ * @event on-copied
+ */
+@localized()
+export class CopyButton extends LitElement {
+  @property({ type: String })
+  value?: string;
+
+  @state()
+  private isCopied: boolean = false;
+
+  timeoutId?: number;
+
+  disconnectedCallback() {
+    window.clearTimeout(this.timeoutId);
+  }
+
+  render() {
+    return html`
+      <sl-button size="small" @click=${this.onClick} ?disabled=${!this.value}
+        >${this.isCopied ? msg("Copied") : msg("Copy")}</sl-button
+      >
+    `;
+  }
+
+  private onClick() {
+    navigator.clipboard.writeText(this.value!);
+
+    this.isCopied = true;
+
+    this.dispatchEvent(new CustomEvent("on-copied", { detail: this.value }));
+
+    this.timeoutId = window.setTimeout(() => {
+      this.isCopied = false;
+    }, 3000);
+  }
+}

frontend/src/components/index.ts

@@ -9,6 +9,9 @@ import("./account-settings").then(({ AccountSettings }) => {
 import("./archive-invite-form").then(({ ArchiveInviteForm }) => {
   customElements.define("btrix-archive-invite-form", ArchiveInviteForm);
 });
+import("./copy-button").then(({ CopyButton }) => {
+  customElements.define("btrix-copy-button", CopyButton);
+});
 import("./invite-form").then(({ InviteForm }) => {
   customElements.define("btrix-invite-form", InviteForm);
 });

frontend/src/pages/archive/crawl-templates.ts

@@ -6,16 +6,30 @@ import type { AuthState } from "../../utils/AuthService";
 import LiteElement, { html } from "../../utils/LiteElement";
 import { getLocaleTimeZone } from "../../utils/localization";
 
-type CrawlTemplate = any; // TODO
+type CrawlTemplate = {
+  id?: string;
+  name: string;
+  schedule: string;
+  runNow: boolean;
+  crawlTimeout?: number;
+  config: {
+    seeds: string[];
+    scopeType?: string;
+    limit?: number;
+  };
+};
 
 const initialValues = {
-  name: `Example crawl ${Date.now()}`, // TODO remove placeholder
+  name: "",
   runNow: true,
-  // crawlTimeoutMinutes: 0,
-  seedUrls: "",
-  scopeType: "prefix",
-  // limit: 0,
+  schedule: "@weekly",
+  config: {
+    seeds: [],
+    scopeType: "prefix",
+    limit: 0,
+  },
 };
+const initialSeedsJson = JSON.stringify(initialValues.config, null, 2);
 const hours = Array.from({ length: 12 }).map((x, i) => ({
   value: i + 1,
   label: `${i + 1}`,
@@ -54,6 +68,21 @@ export class CrawlTemplates extends LiteElement {
       period: "AM",
     };
 
+  @state()
+  private isSeedsJsonView: boolean = false;
+
+  @state()
+  private seedsJson: string = initialSeedsJson;
+
+  @state()
+  private invalidSeedsJsonMessage: string = "";
+
+  @state()
+  private isSubmitting: boolean = false;
+
+  @state()
+  private serverError?: string;
+
   private get timeZone() {
     return Intl.DateTimeFormat().resolvedOptions().timeZone;
   }
@@ -62,17 +91,10 @@ export class CrawlTemplates extends LiteElement {
     return getLocaleTimeZone();
   }
 
-  render() {
-    if (this.isNew) {
-      return this.renderNew();
-    }
-
-    return this.renderList();
-  }
-
-  private renderNew() {
+  private get nextScheduledCrawlMessage() {
     const utcSchedule = this.getUTCSchedule();
-    const nextScheduledCrawlMessage = this.scheduleInterval
+
+    return this.scheduleInterval
       ? msg(html`Next scheduled crawl:
           <sl-format-date
             date="${cronParser
@@ -91,7 +113,17 @@ export class CrawlTemplates extends LiteElement {
             time-zone=${this.timeZone}
           ></sl-format-date>`)
       : undefined;
+  }
 
+  render() {
+    if (this.isNew) {
+      return this.renderNew();
+    }
+
+    return this.renderList();
+  }
+
+  private renderNew() {
     return html`
       <h2 class="text-xl font-bold">${msg("New Crawl Template")}</h2>
       <p>
@@ -101,184 +133,31 @@ export class CrawlTemplates extends LiteElement {
       </p>
 
       <main class="mt-4">
-        <sl-form @sl-submit=${this.onSubmit}>
-          <div class="border rounded-lg md:grid grid-cols-4">
-            <div class="col-span-1 p-4 md:p-8 md:border-b">
-              <h3 class="text-lg font-medium">${msg("Basic settings")}</h3>
+        <div class="border rounded-lg">
+          <sl-form @sl-submit=${this.onSubmit} aria-describedby="formError">
+            <div class="md:grid grid-cols-4">
+              ${this.renderBasicSettings()} ${this.renderPagesSettings()}
             </div>
-            <section class="col-span-3 p-4 md:p-8 border-b grid gap-5">
-              <div>
-                <sl-input
-                  name="name"
-                  label=${msg("Name")}
-                  placeholder=${msg("Example (example.com) Weekly Crawl", {
-                    desc: "Example crawl template name",
-                  })}
-                  autocomplete="off"
-                  value=${initialValues.name}
-                  required
-                ></sl-input>
-              </div>
-              <div>
-                <div class="flex items-end">
-                  <div class="pr-2 flex-1">
-                    <sl-select
-                      name="schedule"
-                      label=${msg("Schedule")}
-                      value=${this.scheduleInterval}
-                      @sl-select=${(e: any) =>
-                        (this.scheduleInterval = e.target.value)}
-                    >
-                      <sl-menu-item value="">${msg("None")}</sl-menu-item>
-                      <sl-menu-item value="daily">${msg("Daily")}</sl-menu-item>
-                      <sl-menu-item value="weekly"
-                        >${msg("Weekly")}</sl-menu-item
-                      >
-                      <sl-menu-item value="monthly"
-                        >${msg("Monthly")}</sl-menu-item
-                      >
-                    </sl-select>
-                  </div>
-                  <div class="grid grid-flow-col gap-2 items-center">
-                    <span class="px-1">${msg("at")}</span>
-                    <sl-select
-                      name="scheduleHour"
-                      value=${this.scheduleTime.hour}
-                      class="w-24"
-                      ?disabled=${!this.scheduleInterval}
-                      @sl-select=${(e: any) =>
-                        (this.scheduleTime = {
-                          ...this.scheduleTime,
-                          hour: +e.target.value,
-                        })}
-                    >
-                      ${hours.map(
-                        ({ value, label }) =>
-                          html`<sl-menu-item value=${value}
-                            >${label}</sl-menu-item
-                          >`
-                      )}
-                    </sl-select>
-                    <span>:</span>
-                    <sl-select
-                      name="scheduleMinute"
-                      value=${this.scheduleTime.minute}
-                      class="w-24"
-                      ?disabled=${!this.scheduleInterval}
-                      @sl-select=${(e: any) =>
-                        (this.scheduleTime = {
-                          ...this.scheduleTime,
-                          minute: +e.target.value,
-                        })}
-                    >
-                      ${minutes.map(
-                        ({ value, label }) =>
-                          html`<sl-menu-item value=${value}
-                            >${label}</sl-menu-item
-                          >`
-                      )}
-                    </sl-select>
-                    <sl-select
-                      value="AM"
-                      class="w-24"
-                      ?disabled=${!this.scheduleInterval}
-                      @sl-select=${(e: any) =>
-                        (this.scheduleTime = {
-                          ...this.scheduleTime,
-                          period: e.target.value,
-                        })}
-                    >
-                      <sl-menu-item value="AM"
-                        >${msg("AM", { desc: "Time AM/PM" })}</sl-menu-item
-                      >
-                      <sl-menu-item value="PM"
-                        >${msg("PM", { desc: "Time AM/PM" })}</sl-menu-item
-                      >
-                    </sl-select>
-                    <span class="px-1">${this.timeZoneShortName}</span>
-                  </div>
-                </div>
-                <div class="text-sm text-gray-500 mt-1">
-                  ${nextScheduledCrawlMessage || msg("No crawls scheduled")}
-                </div>
-              </div>
+
+            <div class="p-4 md:p-8 text-center grid gap-5">
+              ${this.serverError
+                ? html`<btrix-alert id="formError" type="danger"
+                    >${this.serverError}</btrix-alert
+                  >`
+                : ""}
 
               <div>
-                <sl-switch
-                  name="runNow"
-                  ?checked=${initialValues.runNow}
-                  @sl-change=${(e: any) => (this.isRunNow = e.target.checked)}
-                  >${msg("Run immediately on save")}</sl-switch
+                <sl-button
+                  type="primary"
+                  submit
+                  ?loading=${this.isSubmitting}
+                  ?disabled=${this.isSubmitting}
+                  >${msg("Save Crawl Template")}</sl-button
                 >
               </div>
 
-              <div>
-                <sl-input
-                  name="crawlTimeoutMinutes"
-                  label=${msg("Time limit")}
-                  placeholder=${msg("unlimited")}
-                  type="number"
-                >
-                  <span slot="suffix">${msg("minutes")}</span>
-                </sl-input>
-              </div>
-            </section>
-
-            <div class="col-span-1 p-4 md:p-8 md:border-b">
-              <h3 class="text-lg font-medium">${msg("Pages")}</h3>
-            </div>
-            <section class="col-span-3 p-4 md:p-8 border-b grid gap-5">
-              <div>
-                <sl-textarea
-                  name="seedUrls"
-                  label=${msg("Seed URLs")}
-                  helpText=${msg("Separated by a new line, space or comma")}
-                  placeholder=${msg(
-                    `https://webrecorder.net\nhttps://example.com`,
-                    {
-                      desc: "Example seed URLs",
-                    }
-                  )}
-                  help-text=${msg(
-                    "Separate URLs with a new line, space or comma."
-                  )}
-                  rows="3"
-                  value=${initialValues.seedUrls}
-                  required
-                ></sl-textarea>
-              </div>
-              <div>
-                <sl-select
-                  name="scopeType"
-                  label=${msg("Scope type")}
-                  value=${initialValues.scopeType}
-                >
-                  <sl-menu-item value="page">Page</sl-menu-item>
-                  <sl-menu-item value="page-spa">Page SPA</sl-menu-item>
-                  <sl-menu-item value="prefix">Prefix</sl-menu-item>
-                  <sl-menu-item value="host">Host</sl-menu-item>
-                  <sl-menu-item value="any">Any</sl-menu-item>
-                </sl-select>
-              </div>
-              <div>
-                <sl-input
-                  name="limit"
-                  label=${msg("Page limit")}
-                  type="number"
-                  placeholder=${msg("unlimited")}
-                >
-                  <span slot="suffix">${msg("pages")}</span>
-                </sl-input>
-              </div>
-            </section>
-
-            <div class="col-span-4 p-4 md:p-8 text-center">
-              <sl-button type="primary" submit
-                >${msg("Save Crawl Template")}</sl-button
-              >
-
               ${this.isRunNow || this.scheduleInterval
-                ? html`<div class="text-sm text-gray-500 mt-6">
+                ? html`<div class="text-sm text-gray-500">
                     ${this.isRunNow
                       ? html`
                           <p class="mb-2">
@@ -286,12 +165,12 @@ export class CrawlTemplates extends LiteElement {
                           </p>
                         `
                       : ""}
-                    ${nextScheduledCrawlMessage}
+                    ${this.nextScheduledCrawlMessage}
                   </div>`
                 : ""}
             </div>
-          </div>
-        </sl-form>
+          </sl-form>
+        </div>
       </main>
     `;
   }
@@ -316,28 +195,321 @@ export class CrawlTemplates extends LiteElement {
     `;
   }
 
-  private async onSubmit(event: { detail: { formData: FormData } }) {
-    if (!this.authState) return;
+  private renderBasicSettings() {
+    return html`
+      <div class="col-span-1 p-4 md:p-8 md:border-b">
+        <h3 class="text-lg font-medium">${msg("Basic settings")}</h3>
+      </div>
+      <section class="col-span-3 p-4 md:p-8 border-b grid gap-5">
+        <div>
+          <sl-input
+            name="name"
+            label=${msg("Name")}
+            placeholder=${msg("Example (example.com) Weekly Crawl", {
+              desc: "Example crawl template name",
+            })}
+            autocomplete="off"
+            value=${initialValues.name}
+            required
+          ></sl-input>
+        </div>
+        <div>
+          <div class="flex items-end">
+            <div class="pr-2 flex-1">
+              <sl-select
+                name="schedule"
+                label=${msg("Schedule")}
+                value=${this.scheduleInterval}
+                @sl-select=${(e: any) =>
+                  (this.scheduleInterval = e.target.value)}
+              >
+                <sl-menu-item value="">${msg("None")}</sl-menu-item>
+                <sl-menu-item value="daily">${msg("Daily")}</sl-menu-item>
+                <sl-menu-item value="weekly">${msg("Weekly")}</sl-menu-item>
+                <sl-menu-item value="monthly">${msg("Monthly")}</sl-menu-item>
+              </sl-select>
+            </div>
+            <div class="grid grid-flow-col gap-2 items-center">
+              <span class="px-1">${msg("at")}</span>
+              <sl-select
+                name="scheduleHour"
+                value=${this.scheduleTime.hour}
+                class="w-24"
+                ?disabled=${!this.scheduleInterval}
+                @sl-select=${(e: any) =>
+                  (this.scheduleTime = {
+                    ...this.scheduleTime,
+                    hour: +e.target.value,
+                  })}
+              >
+                ${hours.map(
+                  ({ value, label }) =>
+                    html`<sl-menu-item value=${value}>${label}</sl-menu-item>`
+                )}
+              </sl-select>
+              <span>:</span>
+              <sl-select
+                name="scheduleMinute"
+                value=${this.scheduleTime.minute}
+                class="w-24"
+                ?disabled=${!this.scheduleInterval}
+                @sl-select=${(e: any) =>
+                  (this.scheduleTime = {
+                    ...this.scheduleTime,
+                    minute: +e.target.value,
+                  })}
+              >
+                ${minutes.map(
+                  ({ value, label }) =>
+                    html`<sl-menu-item value=${value}>${label}</sl-menu-item>`
+                )}
+              </sl-select>
+              <sl-select
+                value="AM"
+                class="w-24"
+                ?disabled=${!this.scheduleInterval}
+                @sl-select=${(e: any) =>
+                  (this.scheduleTime = {
+                    ...this.scheduleTime,
+                    period: e.target.value,
+                  })}
+              >
+                <sl-menu-item value="AM"
+                  >${msg("AM", { desc: "Time AM/PM" })}</sl-menu-item
+                >
+                <sl-menu-item value="PM"
+                  >${msg("PM", { desc: "Time AM/PM" })}</sl-menu-item
+                >
+              </sl-select>
+              <span class="px-1">${this.timeZoneShortName}</span>
+            </div>
+          </div>
+          <div class="text-sm text-gray-500 mt-1">
+            ${this.nextScheduledCrawlMessage || msg("No crawls scheduled")}
+          </div>
+        </div>
 
-    const { formData } = event.detail;
+        <div>
+          <sl-switch
+            name="runNow"
+            ?checked=${initialValues.runNow}
+            @sl-change=${(e: any) => (this.isRunNow = e.target.checked)}
+            >${msg("Run immediately on save")}</sl-switch
+          >
+        </div>
 
+        <div>
+          <sl-input
+            name="crawlTimeoutMinutes"
+            label=${msg("Time limit")}
+            placeholder=${msg("unlimited")}
+            type="number"
+          >
+            <span slot="suffix">${msg("minutes")}</span>
+          </sl-input>
+        </div>
+      </section>
+    `;
+  }
+
+  private renderPagesSettings() {
+    return html`
+      <div class="col-span-1 p-4 md:p-8 md:border-b">
+        <h3 class="text-lg font-medium">${msg("Crawl configuration")}</h3>
+      </div>
+      <section class="col-span-3 p-4 md:p-8 border-b grid gap-5">
+        <div class="flex justify-between">
+          <h4 class="font-medium">
+            ${this.isSeedsJsonView
+              ? msg("Custom Config")
+              : msg("Configure Seeds")}
+          </h4>
+          <sl-switch
+            ?checked=${this.isSeedsJsonView}
+            @sl-change=${(e: any) => (this.isSeedsJsonView = e.target.checked)}
+          >
+            <span class="text-sm">${msg("Use JSON Editor")}</span>
+          </sl-switch>
+        </div>
+
+        ${this.isSeedsJsonView
+          ? this.renderSeedsJson()
+          : this.renderSeedsForm()}
+      </section>
+    `;
+  }
+
+  private renderSeedsForm() {
+    return html`
+      <sl-textarea
+        name="seedUrls"
+        label=${msg("Seed URLs")}
+        helpText=${msg("Separated by a new line, space or comma")}
+        placeholder=${msg(`https://webrecorder.net\nhttps://example.com`, {
+          desc: "Example seed URLs",
+        })}
+        help-text=${msg("Separate URLs with a new line, space or comma.")}
+        rows="3"
+        required
+      ></sl-textarea>
+      <sl-select
+        name="scopeType"
+        label=${msg("Scope type")}
+        value=${initialValues.config.scopeType}
+      >
+        <sl-menu-item value="page">Page</sl-menu-item>
+        <sl-menu-item value="page-spa">Page SPA</sl-menu-item>
+        <sl-menu-item value="prefix">Prefix</sl-menu-item>
+        <sl-menu-item value="host">Host</sl-menu-item>
+        <sl-menu-item value="any">Any</sl-menu-item>
+      </sl-select>
+      <sl-input
+        name="limit"
+        label=${msg("Page limit")}
+        type="number"
+        placeholder=${msg("unlimited")}
+      >
+        <span slot="suffix">${msg("pages")}</span>
+      </sl-input>
+    `;
+  }
+
+  private renderSeedsJson() {
+    return html`
+      <div class="grid gap-4">
+        <div>
+          <p class="mb-2">
+            ${msg(
+              html`See
+                <a
+                  href="https://github.com/webrecorder/browsertrix-crawler#crawling-configuration-options"
+                  class="text-primary hover:underline"
+                  target="_blank"
+                  >Browsertrix Crawler docs
+                  <sl-icon name="box-arrow-up-right"></sl-icon
+                ></a>
+                for all configuration options.`
+            )}
+          </p>
+        </div>
+
+        <div class="grid grid-cols-3 gap-4">
+          <div class="relative col-span-2">
+            ${this.renderSeedsJsonInput()}
+
+            <div class="absolute top-2 right-2">
+              <btrix-copy-button .value=${this.seedsJson}></btrix-copy-button>
+            </div>
+          </div>
+
+          <div class="col-span-1">
+            ${this.invalidSeedsJsonMessage
+              ? html`<btrix-alert type="danger">
+                  ${this.invalidSeedsJsonMessage}
+                </btrix-alert> `
+              : html` <btrix-alert> ${msg("Valid JSON")} </btrix-alert>`}
+          </div>
+        </div>
+      </div>
+    `;
+  }
+
+  private renderSeedsJsonInput() {
+    return html`
+      <textarea
+        id="json-editor"
+        class="language-json block w-full bg-gray-800 text-gray-50 p-4 rounded font-mono text-sm"
+        autocomplete="off"
+        rows="10"
+        spellcheck="false"
+        .value=${this.seedsJson}
+        @keydown=${(e: any) => {
+          // Add indentation when pressing tab key instead of moving focus
+          if (e.keyCode === /* tab: */ 9) {
+            e.preventDefault();
+
+            const textarea = e.target;
+
+            textarea.setRangeText(
+              "  ",
+              textarea.selectionStart,
+              textarea.selectionStart,
+              "end"
+            );
+          }
+        }}
+        @change=${(e: any) => (this.seedsJson = e.target.value)}
+        @blur=${this.updateSeedsJson}
+      ></textarea>
+    `;
+  }
+
+  private updateSeedsJson(e: any) {
+    const textarea = e.target;
+    const text = textarea.value;
+
+    try {
+      const json = JSON.parse(text);
+
+      this.seedsJson = JSON.stringify(json, null, 2);
+      this.invalidSeedsJsonMessage = "";
+
+      textarea.setCustomValidity("");
+      textarea.reportValidity();
+    } catch (e: any) {
+      this.invalidSeedsJsonMessage = e.message
+        ? msg(str`JSON is invalid: ${e.message.replace("JSON.parse: ", "")}`)
+        : msg("JSON is invalid.");
+    }
+  }
+
+  private parseTemplate(formData: FormData) {
     const crawlTimeoutMinutes = formData.get("crawlTimeoutMinutes");
     const pageLimit = formData.get("limit");
     const seedUrlsStr = formData.get("seedUrls");
-    const params = {
-      name: formData.get("name"),
+    const template: Partial<CrawlTemplate> = {
+      name: formData.get("name") as string,
       schedule: this.getUTCSchedule(),
       runNow: this.isRunNow,
       crawlTimeout: crawlTimeoutMinutes ? +crawlTimeoutMinutes * 60 : 0,
-      config: {
-        seeds: (seedUrlsStr as string).trim().replace(/,/g, " ").split(/\s+/g),
-        scopeType: formData.get("scopeType"),
-        limit: pageLimit ? +pageLimit : 0,
-      },
     };
 
+    if (this.isSeedsJsonView) {
+      template.config = JSON.parse(this.seedsJson);
+    } else {
+      template.config = {
+        seeds: (seedUrlsStr as string).trim().replace(/,/g, " ").split(/\s+/g),
+        scopeType: formData.get("scopeType") as string,
+        limit: pageLimit ? +pageLimit : 0,
+      };
+    }
+
+    return template;
+  }
+
+  private async onSubmit(event: {
+    detail: { formData: FormData };
+    target: any;
+  }) {
+    if (!this.authState) return;
+
+    if (this.isSeedsJsonView && this.invalidSeedsJsonMessage) {
+      // Check JSON validity
+      const jsonEditor = event.target.querySelector("#json-editor");
+
+      jsonEditor.setCustomValidity(msg("Please correct JSON errors."));
+      jsonEditor.reportValidity();
+
+      return;
+    }
+
+    const params = this.parseTemplate(event.detail.formData);
+
     console.log(params);
 
+    this.serverError = undefined;
+    this.isSubmitting = true;
+
     try {
       await this.apiFetch(
         `/archives/${this.archiveId}/crawlconfigs/`,
@@ -348,12 +520,16 @@ export class CrawlTemplates extends LiteElement {
         }
       );
 
-      console.debug("success");
-
       this.navTo(`/archives/${this.archiveId}/crawl-templates`);
-    } catch (e) {
-      console.error(e);
+    } catch (e: any) {
+      if (e?.isApiError) {
+        this.serverError = e?.message;
+      } else {
+        this.serverError = msg("Something unexpected went wrong");
+      }
     }
+
+    this.isSubmitting = false;
   }
 
   /**

frontend/src/shoelace.ts

@@ -8,6 +8,9 @@ import "@shoelace-style/shoelace/dist/components/alert/alert";
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/button/button"
 );
+import(
+  /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/details/details"
+);
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/dialog/dialog"
 );

frontend/src/utils/LiteElement.ts

@@ -55,8 +55,11 @@ export default class LiteElement extends LitElement {
         if (typeof detail === "string") {
           errorMessage = detail;
         } else {
-          // TODO client error details
-          errorMessage = "Unknown API error";
+          // TODO return client error details
+          const fieldDetail = detail[0];
+          const { loc, msg } = fieldDetail;
+
+          errorMessage = `${loc[loc.length - 1]} ${msg}`;
         }
       } catch {
         errorMessage = "Unknown API error";