Re-run crawl from detail view + handle inactive crawl template (#268)

closes #253

frontend/src/pages/archive/crawl-detail.ts

@@ -7,8 +7,7 @@ import { RelativeDuration } from "../../components/relative-duration";
 import type { AuthState } from "../../utils/AuthService";
 import LiteElement, { html } from "../../utils/LiteElement";
 import { CopyButton } from "../../components/copy-button";
-import type { Crawl } from "./types";
+import type { Crawl, CrawlTemplate } from "./types";
-import { times } from "lodash";
 
 type SectionName = "overview" | "watch" | "replay" | "files" | "logs";
 
@@ -42,6 +41,9 @@ export class CrawlDetail extends LiteElement {
   @state()
   private crawl?: Crawl;
 
+  @state()
+  private crawlTemplate?: CrawlTemplate;
+
   @state()
   private sectionName: SectionName = "overview";
 
@@ -77,15 +79,22 @@ export class CrawlDetail extends LiteElement {
     return this.crawl.resources.length > 0;
   }
 
-  async firstUpdated() {
+  firstUpdated() {
     if (!this.crawlsBaseUrl) {
       throw new Error("Crawls base URL not defined");
     }
 
-    this.fetchCrawl();
+    this.fetchData();
   }
 
   updated(changedProperties: Map<string, any>) {
+    const prevId = changedProperties.get("crawlId");
+
+    if (prevId && prevId !== this.crawlId) {
+      // Handle update on URL change, e.g. from re-run
+      this.stopPollTimer();
+      this.fetchData();
+    } else {
       const prevCrawl = changedProperties.get("crawl");
 
       if (prevCrawl && this.crawl) {
@@ -94,6 +103,7 @@ export class CrawlDetail extends LiteElement {
         }
       }
     }
+  }
 
   connectedCallback(): void {
     // Set initial active section based on URL #hash value
@@ -302,6 +312,28 @@ export class CrawlDetail extends LiteElement {
         >
 
         <ul class="text-sm text-0-800 whitespace-nowrap" role="menu">
+          ${!this.isActive && this.crawlTemplate && !this.crawlTemplate.inactive
+            ? html`
+                <li
+                  class="p-2 text-purple-500 hover:bg-purple-500 hover:text-white cursor-pointer"
+                  role="menuitem"
+                  @click=${(e: any) => {
+                    this.runNow();
+                    e.target.closest("sl-dropdown").hide();
+                  }}
+                >
+                  <sl-icon
+                    class="inline-block align-middle"
+                    name="arrow-clockwise"
+                  ></sl-icon>
+                  <span class="inline-block align-middle">
+                    ${msg("Re-run crawl")}
+                  </span>
+                </li>
+                <hr />
+              `
+            : ""}
+
           <li
             class="p-2 hover:bg-zinc-100 cursor-pointer"
             role="menuitem"
@@ -608,6 +640,11 @@ export class CrawlDetail extends LiteElement {
                     <span class="inline-block align-middle">
                       ${this.crawl.configName}
                     </span>
+                    ${this.crawlTemplate?.inactive
+                      ? html`<sl-tag type="warning" size="small"
+                          >${msg("Inactive")}</sl-tag
+                        >`
+                      : ""}
                   </a>
                 `
               : html`<sl-skeleton class="h-6"></sl-skeleton>`}
@@ -743,6 +780,11 @@ export class CrawlDetail extends LiteElement {
     `;
   }
 
+  private async fetchData() {
+    await this.fetchCrawl();
+    this.fetchCrawlTemplate();
+  }
+
   /**
    * Fetch crawl and update internal state
    */
@@ -751,7 +793,8 @@ export class CrawlDetail extends LiteElement {
       this.crawl = await this.getCrawl();
 
       if (this.isActive) {
-        // Start timer for next poll
+        // Restart timer for next poll
+        this.stopPollTimer();
         this.timerId = window.setTimeout(() => {
           this.fetchCrawl();
         }, 1000 * POLL_INTERVAL_SECONDS);
@@ -768,12 +811,6 @@ export class CrawlDetail extends LiteElement {
   }
 
   async getCrawl(): Promise<Crawl> {
-    // Mock to use in dev:
-    // return import("../../__mocks__/api/archives/[id]/crawls").then(
-    //   (module) => module.default.running[0]
-    //   // (module) => module.default.finished[0]
-    // );
-
     const data: Crawl = await this.apiFetch(
       `${this.crawlsAPIBaseUrl || this.crawlsBaseUrl}/${this.crawlId}.json`,
       this.authState!
@@ -782,6 +819,30 @@ export class CrawlDetail extends LiteElement {
     return data;
   }
 
+  /**
+   * Fetch crawl template and update internal state
+   */
+  private async fetchCrawlTemplate(): Promise<void> {
+    try {
+      this.crawlTemplate = await this.getCrawlTemplate();
+    } catch {
+      // Fail silently since page will mostly still function
+    }
+  }
+
+  async getCrawlTemplate(): Promise<CrawlTemplate> {
+    if (!this.crawl) {
+      throw new Error("missing crawl");
+    }
+
+    const data: CrawlTemplate = await this.apiFetch(
+      `/archives/${this.crawl.aid}/crawlconfigs/${this.crawl.cid}`,
+      this.authState!
+    );
+
+    return data;
+  }
+
   private async cancel() {
     if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) {
       const data = await this.apiFetch(
@@ -864,6 +925,63 @@ export class CrawlDetail extends LiteElement {
     this.isSubmittingUpdate = false;
   }
 
+  private async runNow() {
+    if (!this.crawl) return;
+
+    try {
+      // Get crawl config to check if crawl is already running
+      const crawlTemplate = await this.getCrawlTemplate();
+
+      if (crawlTemplate.currCrawlId) {
+        this.notify({
+          message: msg(
+            html`Crawl of <strong>${this.crawl.configName}</strong> is already
+              running.
+              <br />
+              <a
+                class="underline hover:no-underline"
+                href="/archives/${this.crawl
+                  .aid}/crawls/crawl/${crawlTemplate.currCrawlId}"
+                @click=${this.navLink.bind(this)}
+                >View crawl</a
+              >`
+          ),
+          type: "warning",
+          icon: "exclamation-triangle",
+        });
+
+        return;
+      }
+
+      const data = await this.apiFetch(
+        `/archives/${this.crawl.aid}/crawlconfigs/${this.crawl.cid}/run`,
+        this.authState!,
+        {
+          method: "POST",
+        }
+      );
+
+      if (data.started) {
+        this.navTo(`/archives/${this.crawl.aid}/crawls/crawl/${data.started}`);
+      }
+
+      this.notify({
+        message: msg(
+          html`Started crawl from <strong>${this.crawl.configName}</strong>.`
+        ),
+        type: "success",
+        icon: "check2-circle",
+        duration: 8000,
+      });
+    } catch {
+      this.notify({
+        message: msg("Sorry, couldn't run crawl at this time."),
+        type: "danger",
+        icon: "exclamation-octagon",
+      });
+    }
+  }
+
   private stopPollTimer() {
     window.clearTimeout(this.timerId);
   }

frontend/src/pages/archive/crawl-templates-detail.ts

@@ -1125,6 +1125,11 @@ export class CrawlTemplatesDetail extends LiteElement {
         }
       );
 
+      this.crawlTemplate = {
+        ...this.crawlTemplate,
+        inactive: true,
+      };
+
       this.notify({
         message: msg(
           html`Deactivated <strong>${this.crawlTemplate.name}</strong>.`

frontend/src/pages/archive/crawls-list.ts

@@ -11,6 +11,7 @@ import { RelativeDuration } from "../../components/relative-duration";
 import type { AuthState } from "../../utils/AuthService";
 import LiteElement, { html } from "../../utils/LiteElement";
 import type { Crawl, CrawlTemplate } from "./types";
+import type { InitialCrawlTemplate } from "./crawl-templates-new";
 
 type CrawlSearchResult = {
   item: Crawl;
@@ -49,9 +50,6 @@ export class CrawlsList extends LiteElement {
   @property({ type: Object })
   authState!: AuthState;
 
-  @property({ type: String })
-  archiveId?: string;
-
   // e.g. `/archive/${this.archiveId}/crawls`
   @property({ type: String })
   crawlsBaseUrl!: string;
@@ -571,20 +569,17 @@ export class CrawlsList extends LiteElement {
   }
 
   private async runNow(crawl: Crawl) {
-    try {
     // Get crawl config to check if crawl is already running
     const crawlTemplate = await this.getCrawlTemplate(crawl);
 
-      if (crawlTemplate.currCrawlId) {
+    if (crawlTemplate?.currCrawlId) {
       this.notify({
         message: msg(
-            html`Crawl of <strong>${crawl.configName}</strong> is already
+          html`Crawl of <strong>${crawl.configName}</strong> is already running.
-              running.
             <br />
             <a
               class="underline hover:no-underline"
-                href="/archives/${this
+              href="/archives/${crawl.aid}/crawls/crawl/${crawlTemplate.currCrawlId}"
-                  .archiveId}/crawls/crawl/${crawlTemplate.currCrawlId}"
               @click=${this.navLink.bind(this)}
               >View crawl</a
             >`
@@ -596,6 +591,7 @@ export class CrawlsList extends LiteElement {
       return;
     }
 
+    try {
       const data = await this.apiFetch(
         `/archives/${crawl.aid}/crawlconfigs/${crawl.cid}/run`,
         this.authState!,
@@ -614,8 +610,7 @@ export class CrawlsList extends LiteElement {
             <br />
             <a
               class="underline hover:no-underline"
-              href="/archives/${this
+              href="/archives/${crawl.aid}/crawls/crawl/${data.started}#watch"
-                .archiveId}/crawls/crawl/${data.started}#watch"
               @click=${this.navLink.bind(this)}
               >Watch crawl</a
             >`
@@ -624,7 +619,24 @@ export class CrawlsList extends LiteElement {
         icon: "check2-circle",
         duration: 8000,
       });
-    } catch {
+    } catch (e: any) {
+      if (e.isApiError && e.statusCode === 404) {
+        this.notify({
+          message: msg(
+            html`Sorry, cannot rerun crawl from a deactivated crawl template.
+              <br />
+              <button
+                class="underline hover:no-underline"
+                @click=${() => this.duplicateConfig(crawl, crawlTemplate)}
+              >
+                Duplicate crawl template
+              </button>`
+          ),
+          type: "danger",
+          icon: "exclamation-octagon",
+          duration: 8000,
+        });
+      } else {
         this.notify({
           message: msg("Sorry, couldn't run crawl at this time."),
           type: "danger",
@@ -632,6 +644,7 @@ export class CrawlsList extends LiteElement {
         });
       }
     }
+  }
 
   async getCrawlTemplate(crawl: Crawl): Promise<CrawlTemplate> {
     const data: CrawlTemplate = await this.apiFetch(
@@ -641,6 +654,27 @@ export class CrawlsList extends LiteElement {
 
     return data;
   }
+
+  /**
+   * Create a new template using existing template data
+   */
+  private async duplicateConfig(crawl: Crawl, template: CrawlTemplate) {
+    const crawlTemplate: InitialCrawlTemplate = {
+      name: msg(str`${template.name} Copy`),
+      config: template.config,
+      profileid: template.profileid || null,
+    };
+
+    this.navTo(`/archives/${crawl.aid}/crawl-templates/new`, {
+      crawlTemplate,
+    });
+
+    this.notify({
+      message: msg(str`Copied crawl configuration to new template.`),
+      type: "success",
+      icon: "check2-circle",
+    });
+  }
 }
 
 customElements.define("btrix-crawls-list", CrawlsList);