Poll crawls list & add additional details (#116)

frontend/src/pages/archive/crawls-list.ts

@@ -14,8 +14,10 @@ import LiteElement, { html } from "../../utils/LiteElement";
 type Crawl = {
   id: string;
   user: string;
+  username?: string;
   aid: string;
   cid: string;
+  configName?: string;
   schedule: string;
   manual: boolean;
   started: string; // UTC ISO date
@@ -24,6 +26,8 @@ type Crawl = {
   scale: number;
   stats: { done: number; found: number } | null;
   files?: { filename: string; hash: string; size: number }[];
+  fileCount?: number;
+  fileSize?: number;
   completions?: number;
 };
 
@@ -31,12 +35,16 @@ type CrawlSearchResult = {
   item: Crawl;
 };
 
+const POLL_INTERVAL_SECONDS = 10;
 const MIN_SEARCH_LENGTH = 2;
 const sortableFieldLabels = {
   started_desc: msg("Newest"),
   started_asc: msg("Oldest"),
   state: msg("Status"),
+  configName: msg("Crawl Template Name"),
   cid: msg("Crawl Template ID"),
+  fileSize_asc: msg("Smallest Files"),
+  fileSize_desc: msg("Largest Files"),
 };
 
 function isRunning(crawl: Crawl) {
@@ -83,7 +91,16 @@ export class CrawlsList extends LiteElement {
   private filterBy: string = "";
 
   // For fuzzy search:
-  private fuse = new Fuse([], { keys: ["cid"], shouldSort: false });
+  private fuse = new Fuse([], {
+    keys: ["cid", "configName"],
+    shouldSort: false,
+  });
+
+  // For long polling:
+  private timerId?: number;
+
+  // TODO localize
+  private numberFormatter = new Intl.NumberFormat();
 
   private sortCrawls(crawls: CrawlSearchResult[]): CrawlSearchResult[] {
     return orderBy(({ item }) => item[this.orderBy.field])(
@@ -92,11 +109,20 @@ export class CrawlsList extends LiteElement {
   }
 
   protected updated(changedProperties: Map<string, any>) {
-    if (this.shouldFetch && changedProperties.has("shouldFetch")) {
+    if (changedProperties.has("shouldFetch")) {
-      this.fetchCrawls();
+      if (this.shouldFetch) {
+        this.fetchCrawls();
+      } else {
+        this.stopPollTimer();
+      }
     }
   }
 
+  disconnectedCallback(): void {
+    super.disconnectedCallback();
+    this.stopPollTimer();
+  }
+
   render() {
     if (!this.crawls) {
       return html`<div
@@ -137,7 +163,7 @@ export class CrawlsList extends LiteElement {
           <sl-input
             class="w-full"
             slot="trigger"
-            placeholder=${msg("Search by Crawl Template ID")}
+            placeholder=${msg("Search by Crawl Template name or ID")}
             pill
             clearable
             @sl-input=${this.onSearchInput}
@@ -146,7 +172,7 @@ export class CrawlsList extends LiteElement {
           </sl-input>
         </div>
         <div class="col-span-2 md:col-span-1 flex items-center justify-end">
-          <div class="whitespace-nowrap text-sm text-0-600 mr-2">
+          <div class="whitespace-nowrap text-sm text-0-500 mr-2">
             ${msg("Sort by")}
           </div>
           <sl-dropdown
@@ -215,22 +241,40 @@ export class CrawlsList extends LiteElement {
 
   private renderCrawlItem = ({ item: crawl }: CrawlSearchResult) => {
     return html`<li
-      class="grid grid-cols-12 gap-4 md:gap-6 p-4 leading-none border-t first:border-t-0"
+      class="grid grid-cols-12 gap-2 items-center md:gap-6 p-4 leading-none border-t first:border-t-0"
     >
-      <div class="col-span-12 md:col-span-4">
+      <div class="col-span-12 md:col-span-5">
-        <div class="font-medium whitespace-nowrap truncate mb-1">
+        <div class="font-medium mb-1">
-          ${crawl.id}
-        </div>
-        <div class="text-0-500 text-sm whitespace-nowrap truncate">
           <a
-            class="hover:underline"
+            class="hover:text-0-600 transition-colors"
             href=${`/archives/${this.archiveId}/crawl-templates/${crawl.cid}`}
             @click=${this.navLink}
-            >${crawl.cid}</a
+            >${crawl.configName || crawl.cid}</a
           >
         </div>
+        <div class="text-0-700 text-sm whitespace-nowrap truncate">
+          <sl-format-date
+            date=${`${crawl.started}Z` /** Z for UTC */}
+            month="2-digit"
+            day="2-digit"
+            year="2-digit"
+            hour="numeric"
+            minute="numeric"
+          ></sl-format-date>
+          ${crawl.manual
+            ? html` <span
+                class="bg-fuchsia-50 text-fuchsia-700 text-xs rounded px-1 leading-4"
+                >${msg("Manual Start")}</span
+              >`
+            : html`
+                <span
+                  class="bg-teal-50 text-teal-700 text-xs rounded px-1 leading-4"
+                  >${msg("Scheduled Run")}</span
+                >
+              `}
+        </div>
       </div>
-      <div class="col-span-6 md:col-span-3 flex items-start">
+      <div class="col-span-6 md:col-span-2 flex items-start">
         <div class="mr-2">
           <!-- TODO switch case in lit template? needed for tailwindcss purging -->
           <span
@@ -259,7 +303,6 @@ export class CrawlsList extends LiteElement {
               ? html`
                   <sl-relative-time
                     date=${`${crawl.finished}Z` /** Z for UTC */}
-                    sync
                   ></sl-relative-time>
                 `
               : humanizeDuration(
@@ -271,24 +314,60 @@ export class CrawlsList extends LiteElement {
           </div>
         </div>
       </div>
-      <div class="col-span-6 md:col-span-4">
+      <div class="col-span-6 md:col-span-2">
-        <div class="whitespace-nowrap truncate mb-1">
+        ${crawl.finished
-          ${crawl.manual
+          ? html`
-            ? msg(html`Manual start by <span>${crawl.user}</span>`)
+              <div class="whitespace-nowrap truncate text-sm">
-            : msg(html`Scheduled run`)}
+                <span class="font-mono text-0-800 tracking-tighter">
-        </div>
+                  <sl-format-bytes
-
+                    value=${crawl.fileSize || 0}
-        <div class="text-0-500 text-sm whitespace-nowrap truncate">
+                    lang=${/* TODO localize: */ "en"}
-          <sl-format-date
+                  ></sl-format-bytes>
-            class="inline-block align-middle text-0-600"
+                </span>
-            date=${`${crawl.started}Z` /** Z for UTC */}
+                <span class="text-0-500">
-            month="2-digit"
+                  (${crawl.fileCount === 1
-            day="2-digit"
+                    ? msg(str`${crawl.fileCount} file`)
-            year="2-digit"
+                    : msg(str`${crawl.fileCount} files`)})
-            hour="numeric"
+                </span>
-            minute="numeric"
+              </div>
-          ></sl-format-date>
+              <div class="text-0-500 text-sm whitespace-nowrap truncate">
-        </div>
+                ${msg(
+                  str`in ${humanizeDuration(
+                    new Date(`${crawl.finished}Z`).valueOf() -
+                      new Date(`${crawl.started}Z`).valueOf(),
+                    {
+                      secondsDecimalDigits: 0,
+                    }
+                  )}`
+                )}
+              </div>
+            `
+          : crawl.stats
+          ? html`
+              <div
+                class="whitespace-nowrap truncate text-sm text-purple-600 font-mono tracking-tighter"
+              >
+                ${this.numberFormatter.format(crawl.stats.done)}
+                <span class="text-0-400">/</span>
+                ${this.numberFormatter.format(crawl.stats.found)}
+              </div>
+              <div class="text-0-500 text-sm whitespace-nowrap truncate">
+                ${msg("pages crawled")}
+              </div>
+            `
+          : ""}
+      </div>
+      <div class="col-span-6 md:col-span-2">
+        ${crawl.manual
+          ? html`
+              <div class="text-0-500 text-sm whitespace-nowrap truncate">
+                ${msg("Started by")}
+              </div>
+              <div class="text-0-500 text-sm whitespace-nowrap truncate">
+                ${crawl.username || crawl.user}
+              </div>
+            `
+          : ""}
       </div>
       <div class="col-span-12 md:col-span-1 flex justify-end">
         <sl-dropdown>
@@ -337,6 +416,18 @@ export class CrawlsList extends LiteElement {
             >
               ${msg("Copy Crawl Template ID")}
             </li>
+            <li
+              class="p-2 hover:bg-zinc-100 cursor-pointer"
+              role="menuitem"
+              @click=${(e: any) => {
+                e.stopPropagation();
+                this.navTo(
+                  `/archives/${this.archiveId}/crawl-templates/${crawl.cid}`
+                );
+              }}
+            >
+              ${msg("View Crawl Template")}
+            </li>
           </ul>
         </sl-dropdown>
       </div>
@@ -351,12 +442,19 @@ export class CrawlsList extends LiteElement {
    * Fetch crawls and update internal state
    */
   private async fetchCrawls(): Promise<void> {
+    if (!this.shouldFetch) return;
+
     try {
       const { crawls } = await this.getCrawls();
 
       this.crawls = crawls;
       // Update search/filter collection
       this.fuse.setCollection(this.crawls as any);
+
+      // Start timer for next poll
+      this.timerId = window.setTimeout(() => {
+        this.fetchCrawls();
+      }, 1000 * POLL_INTERVAL_SECONDS);
     } catch (e) {
       this.notify({
         message: msg("Sorry, couldn't retrieve crawls at this time."),
@@ -366,6 +464,10 @@ export class CrawlsList extends LiteElement {
     }
   }
 
+  private stopPollTimer() {
+    window.clearTimeout(this.timerId);
+  }
+
   private async getCrawls(): Promise<{ crawls: Crawl[] }> {
     // Mock to use in dev:
     // return import("../../__mocks__/api/archives/[id]/crawls").then(

frontend/src/shoelace.ts

@@ -20,6 +20,9 @@ import(
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/form/form"
 );
+import(
+  /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/format-bytes/format-bytes"
+);
 import(
   /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/format-date/format-date"
 );