Duplicate crawl config from list (#99)

frontend/src/index.ts

@@ -6,12 +6,12 @@ import type { SlDialog } from "@shoelace-style/shoelace";
 import "tailwindcss/tailwind.css";
 
 import type { ArchiveTab } from "./pages/archive";
-import type { NotifyEvent } from "./utils/LiteElement";
+import type { NotifyEvent, NavigateEvent } from "./utils/LiteElement";
 import LiteElement, { html } from "./utils/LiteElement";
 import APIRouter from "./utils/APIRouter";
 import AuthService from "./utils/AuthService";
 import type { LoggedInEvent } from "./utils/AuthService";
-import type { ViewState, NavigateEvent } from "./utils/APIRouter";
+import type { ViewState } from "./utils/APIRouter";
 import type { CurrentUser } from "./types/user";
 import type { AuthState } from "./utils/AuthService";
 import theme from "./theme";
@@ -145,7 +145,7 @@ export class App extends LiteElement {
     }
   }
 
-  navigate(newViewPath: string) {
+  navigate(newViewPath: string, state?: object) {
     if (newViewPath.startsWith("http")) {
       const url = new URL(newViewPath);
       newViewPath = `${url.pathname}${url.search}`;
@@ -158,6 +158,8 @@ export class App extends LiteElement {
       this.viewState = this.router.match(newViewPath);
     }
 
+    this.viewState.data = state;
+
     window.history.pushState(this.viewState, "", this.viewState.pathname);
   }
 
@@ -377,6 +379,7 @@ export class App extends LiteElement {
           @notify="${this.onNotify}"
           .authState=${this.authService.authState}
           .userInfo=${this.userInfo}
+          .viewStateData=${this.viewState.data}
           archiveId=${this.viewState.params.id}
           archiveTab=${this.viewState.params.crawlConfigId
             ? "crawl-templates"
@@ -472,7 +475,7 @@ export class App extends LiteElement {
   onNavigateTo(event: NavigateEvent) {
     event.stopPropagation();
 
-    this.navigate(event.detail);
+    this.navigate(event.detail.url, event.detail.state);
   }
 
   onUserInfoChange(event: CustomEvent<Partial<CurrentUser>>) {

frontend/src/pages/archive/crawl-templates-detail.ts

@@ -109,24 +109,24 @@ export class CrawlTemplatesDetail extends LiteElement {
                           ? " border-t"
                           : ""}"
                         role="row"
-                        title=${seed.url}
+                        title=${typeof seed === "string" ? seed : seed.url}
                       >
                         <div
                           class="col-span-3 break-all leading-tight"
                           role="cell"
                         >
-                          ${seed.url}
+                          ${typeof seed === "string" ? seed : seed.url}
                         </div>
                         <span
                           class="col-span-1 uppercase text-0-500 text-xs"
                           role="cell"
-                          >${seed.scopeType ||
+                          >${(typeof seed !== "string" && seed.scopeType) ||
                           this.crawlTemplate?.config.scopeType}</span
                         >
                         <span
                           class="col-span-1 uppercase text-0-500 text-xs font-mono"
                           role="cell"
-                          >${seed.limit ||
+                          >${(typeof seed !== "string" && seed.limit) ||
                           this.crawlTemplate?.config.limit}</span
                         >
                       </li>`
@@ -290,22 +290,7 @@ export class CrawlTemplatesDetail extends LiteElement {
       this.authState!
     );
 
-    const { config, ...template } = data;
+    return data;
-
-    return {
-      ...template,
-      config: {
-        ...config,
-        // Normalize seed format
-        seeds: config.seeds.map((seed) =>
-          typeof seed === "string"
-            ? {
-                url: seed,
-              }
-            : seed
-        ),
-      },
-    };
   }
 
   private async runNow(): Promise<void> {

frontend/src/pages/archive/crawl-templates-list.ts

@@ -110,9 +110,22 @@ export class CrawlTemplatesList extends LiteElement {
                     style="font-size: 1rem"
                   ></sl-icon-button>
 
-                  <ul role="menu">
+                  <ul class="text-sm whitespace-nowrap" role="menu">
                     <li
-                      class="px-4 py-2 text-danger hover:bg-danger hover:text-white cursor-pointer"
+                      class="p-2 hover:bg-zinc-100 cursor-pointer"
+                      role="menuitem"
+                      @click=${() => this.duplicateConfig(t)}
+                    >
+                      <sl-icon
+                        class="inline-block align-middle px-1"
+                        name="files"
+                      ></sl-icon>
+                      <span class="inline-block align-middle pr-2"
+                        >${msg("Duplicate crawl config")}</span
+                      >
+                    </li>
+                    <li
+                      class="p-2 text-danger hover:bg-danger hover:text-white cursor-pointer"
                       role="menuitem"
                       @click=${(e: any) => {
                         // Close dropdown before deleting template
@@ -121,7 +134,13 @@ export class CrawlTemplatesList extends LiteElement {
                         this.deleteTemplate(t);
                       }}
                     >
-                      ${msg("Delete")}
+                      <sl-icon
+                        class="inline-block align-middle px-1"
+                        name="file-earmark-x"
+                      ></sl-icon>
+                      <span class="inline-block align-middle pr-2"
+                        >${msg("Delete")}</span
+                      >
                     </li>
                   </ul>
                 </sl-dropdown>
@@ -131,14 +150,20 @@ export class CrawlTemplatesList extends LiteElement {
                 <div class="grid gap-2 text-xs leading-none">
                   <div class="overflow-hidden">
                     <sl-tooltip
-                      content=${t.config.seeds.map(({ url }) => url).join(", ")}
+                      content=${t.config.seeds
+                        .map((seed) =>
+                          typeof seed === "string" ? seed : seed.url
+                        )
+                        .join(", ")}
                     >
                       <div
                         class="font-mono whitespace-nowrap truncate text-0-500"
                       >
                         <span class="underline decoration-dashed"
                           >${t.config.seeds
-                            .map(({ url }) => url)
+                            .map((seed) =>
+                              typeof seed === "string" ? seed : seed.url
+                            )
                             .join(", ")}</span
                         >
                       </div>
@@ -251,44 +276,43 @@ export class CrawlTemplatesList extends LiteElement {
    * associated with the crawl templates
    **/
   private async getCrawlTemplates(): Promise<CrawlTemplate[]> {
-    type CrawlConfig = Omit<CrawlTemplate, "config"> & {
+    const data: { crawlConfigs: CrawlTemplate[] } = await this.apiFetch(
-      config: Omit<CrawlTemplate["config"], "seeds"> & {
-        seeds: (string | { url: string })[];
-      };
-    };
-
-    const data: { crawlConfigs: CrawlConfig[] } = await this.apiFetch(
       `/archives/${this.archiveId}/crawlconfigs`,
       this.authState!
     );
 
-    const crawlConfigs: CrawlTemplate[] = [];
     const runningCrawlsMap: RunningCrawlsMap = {};
 
-    data.crawlConfigs.forEach(({ config, ...configMeta }) => {
+    data.crawlConfigs.forEach(({ id, currCrawlId }) => {
-      crawlConfigs.push({
+      if (currCrawlId) {
-        ...configMeta,
+        runningCrawlsMap[id] = currCrawlId;
-        config: {
-          ...config,
-          // Normalize seed format
-          seeds: config.seeds.map((seed) =>
-            typeof seed === "string"
-              ? {
-                  url: seed,
-                }
-              : seed
-          ),
-        },
-      });
-
-      if (configMeta.currCrawlId) {
-        runningCrawlsMap[configMeta.id] = configMeta.currCrawlId;
       }
     });
 
     this.runningCrawlsMap = runningCrawlsMap;
 
-    return crawlConfigs;
+    return data.crawlConfigs;
+  }
+
+  /**
+   * Create a new template using existing template data
+   */
+  private async duplicateConfig(template: CrawlTemplate) {
+    const crawlConfig: CrawlTemplate["config"] = {
+      seeds: template.config.seeds,
+      scopeType: template.config.scopeType,
+      limit: template.config.limit,
+    };
+
+    this.navTo(`/archives/${this.archiveId}/crawl-templates/new`, {
+      crawlConfig,
+    });
+
+    this.notify({
+      message: msg(str`Copied crawl configuration to new template.`),
+      type: "success",
+      icon: "check2-circle",
+    });
   }
 
   private async deleteTemplate(template: CrawlTemplate): Promise<void> {

frontend/src/pages/archive/crawl-templates-new.ts

@@ -1,4 +1,5 @@
 import { state, property } from "lit/decorators.js";
+import { ifDefined } from "lit/directives/if-defined.js";
 import { msg, localized, str } from "@lit/localize";
 import cronParser from "cron-parser";
 
@@ -23,10 +24,8 @@ const initialValues = {
   config: {
     seeds: [],
     scopeType: "prefix",
-    limit: 0,
   },
 };
-const initialSeedsJson = JSON.stringify(initialValues.config, null, 2);
 const hours = Array.from({ length: 12 }).map((x, i) => ({
   value: i + 1,
   label: `${i + 1}`,
@@ -50,6 +49,9 @@ export class CrawlTemplatesNew extends LiteElement {
   @property({ type: String })
   archiveId!: string;
 
+  @property({ type: Object })
+  initialCrawlConfig?: CrawlConfig;
+
   @state()
   private isRunNow: boolean = initialValues.runNow;
 
@@ -69,7 +71,7 @@ export class CrawlTemplatesNew extends LiteElement {
   private isSeedsJsonView: boolean = false;
 
   @state()
-  private seedsJson: string = initialSeedsJson;
+  private seedsJson: string = "";
 
   @state()
   private invalidSeedsJsonMessage: string = "";
@@ -111,6 +113,24 @@ export class CrawlTemplatesNew extends LiteElement {
       : undefined;
   }
 
+  connectedCallback(): void {
+    // Show JSON editor view if complex initial config is specified
+    // (e.g. cloning a template) since form UI doesn't support
+    // all available fields in the config
+    const isComplexConfig = this.initialCrawlConfig?.seeds.some(
+      (seed: any) => typeof seed !== "string"
+    );
+    if (isComplexConfig) {
+      this.isSeedsJsonView = true;
+    }
+    this.initialCrawlConfig = {
+      ...initialValues.config,
+      ...this.initialCrawlConfig,
+    };
+    this.seedsJson = JSON.stringify(this.initialCrawlConfig, null, 2);
+    super.connectedCallback();
+  }
+
   render() {
     return html`
       <h2 class="text-xl font-bold mb-3">${msg("New Crawl Template")}</h2>
@@ -340,12 +360,13 @@ export class CrawlTemplatesNew extends LiteElement {
           "Required. Separate URLs with a new line, space or comma."
         )}
         rows="3"
+        value=${this.initialCrawlConfig!.seeds.join("\n")}
         required
       ></sl-textarea>
       <sl-select
         name="scopeType"
         label=${msg("Scope type")}
-        value=${initialValues.config.scopeType}
+        value=${this.initialCrawlConfig!.scopeType!}
       >
         <sl-menu-item value="page">Page</sl-menu-item>
         <sl-menu-item value="page-spa">Page SPA</sl-menu-item>
@@ -357,6 +378,7 @@ export class CrawlTemplatesNew extends LiteElement {
         name="limit"
         label=${msg("Page limit")}
         type="number"
+        value=${ifDefined(this.initialCrawlConfig!.limit)}
         placeholder=${msg("unlimited")}
       >
         <span slot="suffix">${msg("pages")}</span>

frontend/src/pages/archive/index.ts

@@ -1,6 +1,7 @@
 import { state, property } from "lit/decorators.js";
 import { msg, localized, str } from "@lit/localize";
 
+import type { ViewState } from "../../utils/APIRouter";
 import type { AuthState } from "../../utils/AuthService";
 import type { CurrentUser } from "../../types/user";
 import type { ArchiveData } from "../../utils/archives";
@@ -24,6 +25,9 @@ export class Archive extends LiteElement {
   @property({ type: Object })
   userInfo?: CurrentUser;
 
+  @property({ type: Object })
+  viewStateData?: ViewState["data"];
+
   @property({ type: String })
   archiveId?: string;
 
@@ -144,6 +148,8 @@ export class Archive extends LiteElement {
   }
 
   private renderCrawlTemplates() {
+    const crawlConfig = this.viewStateData?.crawlConfig;
+
     if (this.isNewResourceTab || this.crawlConfigId) {
       return html`
         <div class="md:grid grid-cols-6 gap-6">
@@ -171,6 +177,7 @@ export class Archive extends LiteElement {
                 class="col-span-5 mt-6"
                 .authState=${this.authState!}
                 .archiveId=${this.archiveId!}
+                .initialCrawlConfig=${crawlConfig}
               ></btrix-crawl-templates-new>`}
         </div>
       `;

frontend/src/pages/archive/types.ts

@@ -4,7 +4,7 @@ type SeedConfig = {
 };
 
 export type CrawlConfig = {
-  seeds: ({ url: string } & SeedConfig)[];
+  seeds: (string | ({ url: string } & SeedConfig))[];
 } & SeedConfig;
 
 export type CrawlTemplate = {

frontend/src/utils/APIRouter.ts

@@ -14,8 +14,9 @@ export type ViewState = {
   // e.g. "/users/:id"
   // e.g. "/redirect?url"
   params: { [key: string]: string };
+  // arbitrary data to pass between routes
+  data?: { [key: string]: any };
 };
-export interface NavigateEvent extends CustomEvent {}
 
 export default class APIRouter {
   routes: Routes;

frontend/src/utils/LiteElement.ts

@@ -3,6 +3,13 @@ import { LitElement, html } from "lit";
 import type { Auth } from "../utils/AuthService";
 import { APIError } from "./api";
 
+export interface NavigateEvent extends CustomEvent {
+  detail: {
+    url: string;
+    state?: object;
+  };
+}
+
 export interface NotifyEvent extends CustomEvent {
   detail: {
     /**
@@ -27,21 +34,31 @@ export default class LiteElement extends LitElement {
     return this;
   }
 
-  navTo(url: string) {
+  navTo(url: string, state?: object): void {
-    this.dispatchEvent(
+    const evt: NavigateEvent = new CustomEvent("navigate", {
-      new CustomEvent("navigate", { detail: url, bubbles: true })
+      detail: { url, state },
-    );
+      bubbles: true,
+    });
+    this.dispatchEvent(evt);
   }
 
-  navLink(event: Event) {
+  /**
+   * Bind to anchor tag to prevent full page navigation
+   * @example
+   * ```ts
+   * <a href="/" @click=${this.navLink}>go</a>
+   * ```
+   * @param event Click event
+   */
+  navLink(event: Event): void {
     event.preventDefault();
-    this.dispatchEvent(
+
-      new CustomEvent("navigate", {
+    const evt: NavigateEvent = new CustomEvent("navigate", {
-        detail: (event.currentTarget as HTMLAnchorElement).href,
+      detail: { url: (event.currentTarget as HTMLAnchorElement).href },
       bubbles: true,
       composed: true,
-      })
+    });
-    );
+    this.dispatchEvent(evt);
   }
 
   /**