From fecdc6229dd774654e0ebfe3950b1645fdf812aa Mon Sep 17 00:00:00 2001
From: sua yoo <sua@webrecorder.org>
Date: Thu, 9 Mar 2023 12:18:26 -0800
Subject: [PATCH] Improve crawl queue pagination UX (#680)

* switches to infinite scroll for crawl queue
---
 .../components/crawl-pending-exclusions.ts    | 11 ++-
 frontend/src/components/crawl-queue.ts        | 80 ++++++++++---------
 frontend/src/components/index.ts              |  3 +
 frontend/src/components/numbered-list.ts      |  7 +-
 frontend/src/components/observable.ts         | 50 ++++++++++++
 5 files changed, 109 insertions(+), 42 deletions(-)
 create mode 100644 frontend/src/components/observable.ts

diff --git a/frontend/src/components/crawl-pending-exclusions.ts b/frontend/src/components/crawl-pending-exclusions.ts
index 3bcb7e9a..b41d448f 100644
--- a/frontend/src/components/crawl-pending-exclusions.ts
+++ b/frontend/src/components/crawl-pending-exclusions.ts
@@ -24,7 +24,7 @@ export class CrawlPendingExclusions extends LiteElement {
   private page: number = 1;
 
   @state()
-  private pageSize: number = 30;
+  private pageSize: number = 20;
 
   @state()
   private isOpen: boolean = false;
@@ -51,7 +51,14 @@ export class CrawlPendingExclusions extends LiteElement {
         <span slot="title">
           ${msg("Pending Exclusions")} ${this.renderBadge()}
         </span>
-        <div slot="summary-description">
+        <div
+          slot="summary-description"
+          @click=${(e: MouseEvent) => {
+            // Prevent toggle when clicking pagination
+            e.stopPropagation();
+            e.preventDefault();
+          }}
+        >
           ${this.isOpen && this.total && this.total > this.pageSize
             ? html`<btrix-pagination
                 size=${this.pageSize}
diff --git a/frontend/src/components/crawl-queue.ts b/frontend/src/components/crawl-queue.ts
index 018ba5c3..359b669c 100644
--- a/frontend/src/components/crawl-queue.ts
+++ b/frontend/src/components/crawl-queue.ts
@@ -1,5 +1,7 @@
 import { property, state } from "lit/decorators.js";
 import { msg, localized, str } from "@lit/localize";
+import { when } from "lit/directives/when.js";
+import throttle from "lodash/fp/throttle";
 
 import LiteElement, { html } from "../utils/LiteElement";
 import type { AuthState } from "../utils/AuthService";
@@ -51,10 +53,7 @@ export class CrawlQueue extends LiteElement {
   private isLoading = false;
 
   @state()
-  private page: number = 1;
-
-  @state()
-  private pageSize: number = 30;
+  private pageSize: number = 50;
 
   private timerId?: number;
 
@@ -68,7 +67,7 @@ export class CrawlQueue extends LiteElement {
       changedProperties.has("authState") ||
       changedProperties.has("orgId") ||
       changedProperties.has("crawlId") ||
-      changedProperties.has("page") ||
+      changedProperties.has("pageSize") ||
       changedProperties.has("regex")
     ) {
       this.fetchOnUpdate();
@@ -77,23 +76,10 @@ export class CrawlQueue extends LiteElement {
 
   render() {
     return html`
-      <btrix-details open>
-        <span slot="title"> ${msg("Crawl Queue")} ${this.renderBadge()} </span>
-        <div slot="summary-description">
-          ${this.queue?.total && this.queue.total > this.pageSize
-            ? html`<btrix-pagination
-                size=${this.pageSize}
-                totalCount=${this.queue.total}
-                @page-change=${(e: CustomEvent) => {
-                  this.page = e.detail.page;
-                }}
-              >
-              </btrix-pagination>`
-            : ""}
-        </div>
-
-        ${this.renderContent()}
-      </btrix-details>
+      <btrix-section-heading style="--margin: var(--sl-spacing-small)"
+        >${msg("Crawl Queue")} ${this.renderBadge()}</btrix-section-heading
+      >
+      ${this.renderContent()}
     `;
   }
 
@@ -112,6 +98,8 @@ export class CrawlQueue extends LiteElement {
       `;
     }
 
+    if (!this.queue) return;
+
     const excludedURLStyles = [
       "--marker-color: var(--sl-color-danger-500)",
       "--link-color: var(--sl-color-danger-500)",
@@ -121,9 +109,9 @@ export class CrawlQueue extends LiteElement {
     return html`
       <btrix-numbered-list
         class="text-xs break-all"
-        .items=${this.queue?.results.map((url, idx) => ({
-          order: idx + 1 + (this.page - 1) * this.pageSize,
-          style: this.queue?.matched.some((v) => v === url)
+        .items=${this.queue.results.map((url, idx) => ({
+          order: idx + 1,
+          style: this.queue!.matched.some((v) => v === url)
             ? excludedURLStyles
             : "",
           content: html`<a
@@ -137,17 +125,24 @@ export class CrawlQueue extends LiteElement {
       ></btrix-numbered-list>
 
       <footer class="text-center">
-        <span class="text-xs text-neutral-400" aria-live="polite">
-          ${msg(
-            str`${(
-              (this.page - 1) * this.pageSize +
-              1
-            ).toLocaleString()}⁠–⁠${Math.min(
-              this.page * this.pageSize,
-              this.queue.total
-            ).toLocaleString()} of ${this.queue.total.toLocaleString()} URLs`
-          )}
-        </span>
+        ${when(
+          this.queue.total === this.queue.results.length,
+          () =>
+            html`<div class="text-xs text-neutral-400 py-3">
+              ${msg("End of queue")}
+            </div>`,
+          () => html`
+            <btrix-observable @intersect=${this.onLoadMoreIntersect}>
+              <div class="py-3">
+                <sl-icon-button
+                  name="three-dots"
+                  @click=${this.loadMore}
+                  label=${msg("Load more")}
+                ></sl-icon-button>
+              </div>
+            </btrix-observable>
+          `
+        )}
       </footer>
     `;
   }
@@ -176,6 +171,15 @@ export class CrawlQueue extends LiteElement {
     `;
   }
 
+  private onLoadMoreIntersect = throttle(50)((e: CustomEvent) => {
+    if (!e.detail.entry.isIntersecting) return;
+    this.loadMore();
+  });
+
+  private loadMore() {
+    this.pageSize = this.pageSize + 50;
+  }
+
   private async fetchOnUpdate() {
     window.clearInterval(this.timerId);
     await this.performUpdate;
@@ -201,9 +205,7 @@ export class CrawlQueue extends LiteElement {
 
   private async getQueue(): Promise<ResponseData> {
     const data: ResponseData = await this.apiFetch(
-      `/orgs/${this.orgId}/crawls/${this.crawlId}/queue?offset=${
-        (this.page - 1) * this.pageSize
-      }&count=${this.pageSize}&regex=${this.regex}`,
+      `/orgs/${this.orgId}/crawls/${this.crawlId}/queue?offset=0&count=${this.pageSize}&regex=${this.regex}`,
       this.authState!
     );
 
diff --git a/frontend/src/components/index.ts b/frontend/src/components/index.ts
index 21919d98..77c67097 100644
--- a/frontend/src/components/index.ts
+++ b/frontend/src/components/index.ts
@@ -110,6 +110,9 @@ import("./crawl-status").then(({ CrawlStatus }) => {
 import("./crawl-metadata-editor").then(({ CrawlMetadataEditor }) => {
   customElements.define("btrix-crawl-metadata-editor", CrawlMetadataEditor);
 });
+import("./observable").then(({ Observable }) => {
+  customElements.define("btrix-observable", Observable);
+});
 
 customElements.define("btrix-alert", Alert);
 customElements.define("btrix-input", Input);
diff --git a/frontend/src/components/numbered-list.ts b/frontend/src/components/numbered-list.ts
index 65c55e6d..0aeaf782 100644
--- a/frontend/src/components/numbered-list.ts
+++ b/frontend/src/components/numbered-list.ts
@@ -47,12 +47,17 @@ export class NumberedList extends LitElement {
     }
 
     .item-content {
+      --item-height: 1.5rem;
+      contain: strict;
+      contain-intrinsic-height: auto var(--item-height);
+      content-visibility: auto;
       border-left: var(--sl-panel-border-width) solid
         var(--sl-panel-border-color);
       border-right: var(--sl-panel-border-width) solid
         var(--sl-panel-border-color);
       padding: var(--sl-spacing-2x-small) var(--sl-spacing-x-small);
-      line-height: 1.25;
+      height: var(--item-height);
+      box-sizing: border-box;
     }
 
     li:first-child .item-content {
diff --git a/frontend/src/components/observable.ts b/frontend/src/components/observable.ts
new file mode 100644
index 00000000..b4db0c03
--- /dev/null
+++ b/frontend/src/components/observable.ts
@@ -0,0 +1,50 @@
+import { LitElement, html } from "lit";
+import { query } from "lit/decorators.js";
+
+export type IntersectEvent = CustomEvent<{
+  entry: IntersectionObserverEntry;
+}>;
+
+/**
+ * Observe element with Intersection Obserer API.
+ *
+ * @example Usage:
+ * ```
+ * <btrix-observable @intersect=${console.log}>
+ *   Observe me!
+ * </btrix-observable>
+ * ```
+ *
+ * @event intersect { entry: IntersectionObserverEntry }
+ */
+export class Observable extends LitElement {
+  @query(".target")
+  private target?: HTMLElement;
+
+  private observer?: IntersectionObserver;
+
+  connectedCallback(): void {
+    super.connectedCallback();
+    this.observer = new IntersectionObserver(this.handleIntersect);
+  }
+
+  disconnectedCallback(): void {
+    this.observer?.disconnect();
+  }
+
+  firstUpdated() {
+    this.observer?.observe(this.target!);
+  }
+
+  private handleIntersect = ([entry]: IntersectionObserverEntry[]) => {
+    this.dispatchEvent(
+      <IntersectEvent>new CustomEvent("intersect", {
+        detail: { entry },
+      })
+    );
+  };
+
+  render() {
+    return html`<div class="target"><slot></slot></div>`;
+  }
+}