From b9a24fa5e2597454c5a5e3041320cd855a878894 Mon Sep 17 00:00:00 2001
From: sua yoo <sua@webrecorder.org>
Date: Fri, 17 Mar 2023 21:04:08 -0700
Subject: [PATCH] Combine watch crawl with crawl queue (#710)

- crawl queue and watch page are now part of single view
- exclusions can be edited via 'Edit Exclusions' popup
---
 .../components/crawl-pending-exclusions.ts    |  29 +--
 frontend/src/components/crawl-queue.ts        |   2 +-
 frontend/src/components/exclusion-editor.ts   |  26 +--
 frontend/src/components/numbered-list.ts      |   5 +-
 frontend/src/components/screencast.ts         |  56 ++---
 frontend/src/pages/org/crawl-detail.ts        | 209 +++++++++++-------
 6 files changed, 165 insertions(+), 162 deletions(-)

diff --git a/frontend/src/components/crawl-pending-exclusions.ts b/frontend/src/components/crawl-pending-exclusions.ts
index b41d448f..39763d5c 100644
--- a/frontend/src/components/crawl-pending-exclusions.ts
+++ b/frontend/src/components/crawl-pending-exclusions.ts
@@ -24,10 +24,7 @@ export class CrawlPendingExclusions extends LiteElement {
   private page: number = 1;
 
   @state()
-  private pageSize: number = 20;
-
-  @state()
-  private isOpen: boolean = false;
+  private pageSize: number = 10;
 
   private get total(): number {
     return this.matchedURLs?.length || 0;
@@ -44,22 +41,10 @@ export class CrawlPendingExclusions extends LiteElement {
 
   render() {
     return html`
-      <btrix-details
-        ?open=${this.isOpen}
-        @on-toggle=${(e: CustomEvent) => (this.isOpen = e.detail.open)}
-      >
-        <span slot="title">
-          ${msg("Pending Exclusions")} ${this.renderBadge()}
-        </span>
-        <div
-          slot="summary-description"
-          @click=${(e: MouseEvent) => {
-            // Prevent toggle when clicking pagination
-            e.stopPropagation();
-            e.preventDefault();
-          }}
-        >
-          ${this.isOpen && this.total && this.total > this.pageSize
+      <btrix-section-heading style="--margin: var(--sl-spacing-small)">
+        <div class="flex items-center justify-between">
+          <div>${msg("Pending Exclusions")} ${this.renderBadge()}</div>
+          ${this.total && this.total > this.pageSize
             ? html`<btrix-pagination
                 size=${this.pageSize}
                 totalCount=${this.total}
@@ -70,8 +55,8 @@ export class CrawlPendingExclusions extends LiteElement {
               </btrix-pagination>`
             : ""}
         </div>
-        ${this.renderContent()}
-      </btrix-details>
+      </btrix-section-heading>
+      ${this.renderContent()}
     `;
   }
 
diff --git a/frontend/src/components/crawl-queue.ts b/frontend/src/components/crawl-queue.ts
index 359b669c..c65659b9 100644
--- a/frontend/src/components/crawl-queue.ts
+++ b/frontend/src/components/crawl-queue.ts
@@ -77,7 +77,7 @@ export class CrawlQueue extends LiteElement {
   render() {
     return html`
       <btrix-section-heading style="--margin: var(--sl-spacing-small)"
-        >${msg("Crawl Queue")} ${this.renderBadge()}</btrix-section-heading
+        >${msg("Queued URLs")} ${this.renderBadge()}</btrix-section-heading
       >
       ${this.renderContent()}
     `;
diff --git a/frontend/src/components/exclusion-editor.ts b/frontend/src/components/exclusion-editor.ts
index fb3d45a6..f014743a 100644
--- a/frontend/src/components/exclusion-editor.ts
+++ b/frontend/src/components/exclusion-editor.ts
@@ -79,13 +79,17 @@ export class ExclusionEditor extends LiteElement {
 
   render() {
     return html`
-      ${this.renderTable()}
-      ${this.isActiveCrawl && this.regex
-        ? html` <section class="mt-5">${this.renderPending()}</section> `
-        : ""}
-      ${this.isActiveCrawl
-        ? html` <section class="mt-5">${this.renderQueue()}</section> `
-        : ""}
+      <div class="grid gap-6 grid-cols-1 lg:grid-cols-2">
+        <div class="col-span-1">${this.renderTable()}</div>
+        <div class="col-span-1">
+          ${this.isActiveCrawl && this.regex
+            ? html` <section class="mt-5">${this.renderPending()}</section> `
+            : ""}
+          ${this.isActiveCrawl
+            ? html` <section class="mt-5">${this.renderQueue()}</section> `
+            : ""}
+        </div>
+      </div>
     `;
   }
 
@@ -164,9 +168,7 @@ export class ExclusionEditor extends LiteElement {
           icon: "check2-circle",
         });
 
-        this.dispatchEvent(
-          new CustomEvent("on-success")
-        );
+        this.dispatchEvent(new CustomEvent("on-success"));
       } else {
         throw data;
       }
@@ -239,9 +241,7 @@ export class ExclusionEditor extends LiteElement {
         await this.updateComplete;
 
         onSuccess();
-        this.dispatchEvent(
-          new CustomEvent("on-success")
-        );
+        this.dispatchEvent(new CustomEvent("on-success"));
       } else {
         throw data;
       }
diff --git a/frontend/src/components/numbered-list.ts b/frontend/src/components/numbered-list.ts
index 0aeaf782..ee1922e3 100644
--- a/frontend/src/components/numbered-list.ts
+++ b/frontend/src/components/numbered-list.ts
@@ -48,7 +48,7 @@ export class NumberedList extends LitElement {
 
     .item-content {
       --item-height: 1.5rem;
-      contain: strict;
+      contain: paint;
       contain-intrinsic-height: auto var(--item-height);
       content-visibility: auto;
       border-left: var(--sl-panel-border-width) solid
@@ -56,7 +56,8 @@ export class NumberedList extends LitElement {
       border-right: var(--sl-panel-border-width) solid
         var(--sl-panel-border-color);
       padding: var(--sl-spacing-2x-small) var(--sl-spacing-x-small);
-      height: var(--item-height);
+      line-height: 1;
+      min-height: var(--item-height);
       box-sizing: border-box;
     }
 
diff --git a/frontend/src/components/screencast.ts b/frontend/src/components/screencast.ts
index 49629402..68c8e4da 100644
--- a/frontend/src/components/screencast.ts
+++ b/frontend/src/components/screencast.ts
@@ -53,35 +53,22 @@ export class Screencast extends LitElement {
 
     .container {
       display: grid;
-      gap: 0.5rem;
-    }
-
-    .screen-count {
-      color: var(--sl-color-neutral-400);
-      font-size: var(--sl-font-size-small);
-      margin-bottom: var(--sl-spacing-x-small);
-    }
-
-    .screen-count span,
-    .screen-count sl-icon {
-      display: inline-block;
-      vertical-align: middle;
+      gap: 1rem;
     }
 
     .screen {
-      border: 1px solid var(--sl-panel-border-color);
-      border-radius: var(--sl-border-radius-medium);
+      border: 1px solid var(--sl-color-neutral-300);
+      border-radius: var(--sl-border-radius-large);
       overflow: hidden;
     }
 
     .screen[role="button"] {
       cursor: pointer;
-      transition: opacity 0.1s border-color 0.1s;
+      transition: var(--sl-transition-fast) box-shadow;
     }
 
     .screen[role="button"]:hover {
-      opacity: 0.8;
-      border-color: var(--sl-color-neutral-300);
+      box-shadow: var(--sl-shadow-medium);
     }
 
     figure {
@@ -111,7 +98,10 @@ export class Screencast extends LitElement {
     }
 
     .frame {
-      background-color: var(--sl-color-neutral-50);
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      background-color: var(--sl-color-sky-50);
       overflow: hidden;
     }
 
@@ -123,6 +113,10 @@ export class Screencast extends LitElement {
       outline: 0;
       border: 0;
     }
+
+    sl-spinner {
+      font-size: 1.75rem;
+    }
   `;
 
   @property({ type: String })
@@ -190,30 +184,8 @@ export class Screencast extends LitElement {
   }
 
   render() {
-    const browserWindows = this.browsersCount * this.scale;
-
     return html`
       <div class="wrapper">
-        ${!this.dataList.length
-          ? html`<div class="spinner">
-              <sl-spinner></sl-spinner>
-            </div> `
-          : html`
-              <div class="screen-count">
-                <span
-                  >${browserWindows > 1
-                    ? msg(str`Running in ${browserWindows} browser windows`)
-                    : msg(str`Running in 1 browser window`)}</span
-                >
-                <sl-tooltip
-                  content=${msg(
-                    str`${this.browsersCount} browser(s) × ${this.scale} crawler(s). Number of crawlers corresponds to scale.`
-                  )}
-                  ><sl-icon name="info-circle"></sl-icon
-                ></sl-tooltip>
-              </div>
-            `}
-
         <div
           class="container"
           style="grid-template-columns: repeat(${this
@@ -239,7 +211,7 @@ export class Screencast extends LitElement {
                 >
                   ${pageData
                     ? html`<img src="data:image/png;base64,${pageData.data}" />`
-                    : ""}
+                    : html`<sl-spinner></sl-spinner>`}
                 </div>
               </figure>`
           )}
diff --git a/frontend/src/pages/org/crawl-detail.ts b/frontend/src/pages/org/crawl-detail.ts
index a12b9073..4ff78986 100644
--- a/frontend/src/pages/org/crawl-detail.ts
+++ b/frontend/src/pages/org/crawl-detail.ts
@@ -19,7 +19,7 @@ const SECTIONS = [
   "config",
   "exclusions",
 ] as const;
-type SectionName = typeof SECTIONS[number];
+type SectionName = (typeof SECTIONS)[number];
 
 const POLL_INTERVAL_SECONDS = 10;
 
@@ -61,7 +61,7 @@ export class CrawlDetail extends LiteElement {
   private isSubmittingUpdate: boolean = false;
 
   @state()
-  private openDialogName?: "scale" | "metadata";
+  private openDialogName?: "scale" | "metadata" | "exclusions";
 
   @state()
   private isDialogVisible: boolean = false;
@@ -135,10 +135,23 @@ export class CrawlDetail extends LiteElement {
     let sectionContent: string | TemplateResult = "";
 
     switch (this.sectionName) {
+      case "exclusions":
       case "watch": {
         if (this.crawl) {
+          const isRunning = this.crawl.state === "running";
           sectionContent = this.renderPanel(
-            msg("Watch Crawl"),
+            html`<span>${msg("Watch Crawl")}</span>
+              <sl-button
+                size="small"
+                ?disabled=${!isRunning}
+                @click=${() => {
+                  this.openDialogName = "scale";
+                  this.isDialogVisible = true;
+                }}
+              >
+                <sl-icon name="plus-slash-minus" slot="prefix"></sl-icon>
+                <span> ${msg("Crawler Instances")} </span>
+              </sl-button> `,
             this.renderWatch()
           );
         } else {
@@ -163,13 +176,6 @@ export class CrawlDetail extends LiteElement {
       case "logs":
         sectionContent = this.renderPanel(msg("Logs"), this.renderLogs());
         break;
-      case "exclusions":
-        sectionContent = this.renderPanel(
-          msg("Crawl Exclusions"),
-          this.renderExclusions()
-        );
-        break;
-
       case "config":
         sectionContent = this.renderPanel(msg("Config"), this.renderConfig());
         break;
@@ -182,30 +188,28 @@ export class CrawlDetail extends LiteElement {
             <div class="col-span-1 flex flex-col">
               ${this.renderPanel(
                 html`
-                  <div class="flex items-center justify-between">
-                    ${msg("Metadata")}
-                    ${when(
-                      this.isCrawler,
-                      () => html`
-                        <sl-tooltip
-                          content=${msg(
-                            "Metadata cannot be edited while crawl is running."
-                          )}
-                          ?disabled=${!this.isActive}
-                        >
-                          <sl-icon-button
-                            class=${`text-base${
-                              this.isActive ? " cursor-not-allowed" : ""
-                            }`}
-                            name="pencil"
-                            @click=${this.openMetadataEditor}
-                            aria-label=${msg("Edit Metadata")}
-                            ?disabled=${this.isActive}
-                          ></sl-icon-button>
-                        </sl-tooltip>
-                      `
-                    )}
-                  </div>
+                  ${msg("Metadata")}
+                  ${when(
+                    this.isCrawler,
+                    () => html`
+                      <sl-tooltip
+                        content=${msg(
+                          "Metadata cannot be edited while crawl is running."
+                        )}
+                        ?disabled=${!this.isActive}
+                      >
+                        <sl-icon-button
+                          class=${`text-base${
+                            this.isActive ? " cursor-not-allowed" : ""
+                          }`}
+                          name="pencil"
+                          @click=${this.openMetadataEditor}
+                          aria-label=${msg("Edit Metadata")}
+                          ?disabled=${this.isActive}
+                        ></sl-icon-button>
+                      </sl-tooltip>
+                    `
+                  )}
                 `,
                 this.renderMetadata()
               )}
@@ -245,16 +249,6 @@ export class CrawlDetail extends LiteElement {
         </section>
       </main>
 
-      <btrix-dialog
-        label=${msg("Edit Crawler Instances")}
-        ?open=${this.openDialogName === "scale"}
-        @sl-request-close=${() => (this.openDialogName = undefined)}
-        @sl-show=${() => (this.isDialogVisible = true)}
-        @sl-after-hide=${() => (this.isDialogVisible = false)}
-      >
-        ${this.isDialogVisible ? this.renderEditScale() : ""}
-      </btrix-dialog>
-
       <btrix-crawl-metadata-editor
         .authState=${this.authState}
         .crawl=${this.crawl}
@@ -343,12 +337,6 @@ export class CrawlDetail extends LiteElement {
             icon: "info-circle-fill",
             label: msg("Overview"),
           })}
-          ${renderNavItem({
-            section: "exclusions",
-            iconLibrary: "default",
-            icon: "list-ul",
-            label: msg("Crawl Queue & Exclusions"),
-          })}
           ${this.isActive
             ? renderNavItem({
                 section: "watch",
@@ -362,7 +350,7 @@ export class CrawlDetail extends LiteElement {
                 section: "replay",
                 iconLibrary: "app",
                 icon: "link-replay",
-                label: msg("Replay"),
+                label: msg("Replay Crawl"),
               })
             : ""}
           ${!this.isActive
@@ -401,16 +389,6 @@ export class CrawlDetail extends LiteElement {
           ${this.isActive
             ? html`
                 <sl-button-group>
-                  <sl-button
-                    size="small"
-                    @click=${() => {
-                      this.openDialogName = "scale";
-                      this.isDialogVisible = true;
-                    }}
-                  >
-                    <sl-icon name="plus-slash-minus" slot="prefix"></sl-icon>
-                    <span> ${msg("Crawler Instances")} </span>
-                  </sl-button>
                   <sl-button size="small" @click=${this.stop}>
                     <sl-icon name="slash-circle" slot="prefix"></sl-icon>
                     <span> ${msg("Stop")} </span>
@@ -551,7 +529,12 @@ export class CrawlDetail extends LiteElement {
 
   private renderPanel(title: any, content: any) {
     return html`
-      <h2 class="flex-0 text-lg font-semibold mb-2">${title}</h2>
+      <h2
+        id="exclusions"
+        class="flex-0 flex items-center justify-between text-lg font-semibold leading-none h-8 min-h-fit mb-2"
+      >
+        ${title}
+      </h2>
       <div class="flex-1 rounded-lg border p-5">${content}</div>
     `;
   }
@@ -652,33 +635,95 @@ export class CrawlDetail extends LiteElement {
                   </div>
                 `
               : ""}
-
-            <div
-              id="screencast-crawl"
-              class="${isStopping ? "opacity-40" : ""} transition-opacity"
-            >
-              <btrix-screencast
-                authToken=${authToken}
-                orgId=${this.crawl.oid}
-                crawlId=${this.crawlId}
-                scale=${this.crawl.scale}
-              ></btrix-screencast>
-            </div>
           `
         : this.renderInactiveCrawlMessage()}
+      ${when(
+        isRunning,
+        () => html`
+          <div
+            id="screencast-crawl"
+            class="${isStopping ? "opacity-40" : ""} transition-opacity"
+          >
+            <btrix-screencast
+              authToken=${authToken}
+              orgId=${this.crawl!.oid}
+              crawlId=${this.crawlId}
+              scale=${this.crawl!.scale}
+            ></btrix-screencast>
+          </div>
+
+          <section class="mt-8">${this.renderExclusions()}</section>
+
+          <btrix-dialog
+            label=${msg("Edit Crawler Instances")}
+            ?open=${this.openDialogName === "scale"}
+            @sl-request-close=${() => (this.openDialogName = undefined)}
+            @sl-show=${() => (this.isDialogVisible = true)}
+            @sl-after-hide=${() => (this.isDialogVisible = false)}
+          >
+            ${this.isDialogVisible ? this.renderEditScale() : ""}
+          </btrix-dialog>
+        `
+      )}
     `;
   }
 
   private renderExclusions() {
     return html`
-      <btrix-exclusion-editor
-        orgId=${ifDefined(this.crawl?.oid)}
-        crawlId=${ifDefined(this.crawl?.id)}
-        .config=${this.crawl?.config}
-        .authState=${this.authState}
-        ?isActiveCrawl=${this.crawl && this.isActive}
-        @on-success=${this.handleExclusionChange}
-      ></btrix-exclusion-editor>
+      <header class="flex items-center justify-between">
+        <h3 class="leading-none text-lg font-semibold mb-2">
+          ${msg("Crawl URLs")}
+        </h3>
+        <sl-button
+          size="small"
+          variant="primary"
+          @click=${() => {
+            this.openDialogName = "exclusions";
+            this.isDialogVisible = true;
+          }}
+        >
+          <sl-icon slot="prefix" name="table"></sl-icon>
+          ${msg("Edit Exclusions")}
+        </sl-button>
+      </header>
+
+      ${when(
+        this.crawl,
+        () => html`
+          <btrix-crawl-queue
+            orgId=${this.crawl!.oid}
+            crawlId=${this.crawlId}
+            .authState=${this.authState}
+          ></btrix-crawl-queue>
+        `
+      )}
+
+      <btrix-dialog
+        label=${msg("Crawl Queue Editor")}
+        ?open=${this.openDialogName === "exclusions"}
+        style=${/* max-w-screen-lg: */ `--width: 1124px;`}
+        @sl-request-close=${() => (this.openDialogName = undefined)}
+        @sl-show=${() => (this.isDialogVisible = true)}
+        @sl-after-hide=${() => (this.isDialogVisible = false)}
+      >
+        ${this.isDialogVisible
+          ? html`<btrix-exclusion-editor
+              orgId=${ifDefined(this.crawl?.oid)}
+              crawlId=${ifDefined(this.crawl?.id)}
+              .config=${this.crawl?.config}
+              .authState=${this.authState}
+              ?isActiveCrawl=${this.crawl && this.isActive}
+              @on-success=${this.handleExclusionChange}
+            ></btrix-exclusion-editor>`
+          : ""}
+        <div slot="footer">
+          <sl-button
+            size="small"
+            @click=${() => (this.openDialogName = undefined)}
+            >${msg("Done Editing")}</sl-button
+          >
+        </div>
+      </btrix-dialog>
     `;
   }