List web captures in Collection (#1024)

- Adds tab for "Web Captures" in Collection detail view - Move Collection description under Replay section - Fixes app reloading when clicking into a Collection - Standardizes Web Capture list headers from "Finished -> "Created Date"
2023-08-01 09:14:27 -07:00 · 2023-08-01 09:14:27 -07:00 · 54e2b2c703
commit 54e2b2c703
parent 06cf9c7cc3
10 changed files with 284 additions and 78 deletions
--- a/frontend/src/components/button.ts
+++ b/frontend/src/components/button.ts
@ -129,6 +129,7 @@ export class Button extends LitElement {
        raised: this.raised,
      })}
      ?disabled=${this.disabled}
+      href=${ifDefined(this.href)}
      aria-label=${ifDefined(this.label)}
      @click=${this.handleClick}
    >
--- a/frontend/src/components/crawl-list.ts
+++ b/frontend/src/components/crawl-list.ts
@ -226,9 +226,8 @@ export class CrawlListItem extends LitElement {
    return html`<a
      class="item row"
      role="button"
-      href="${this.baseUrl || `/orgs/${this.crawl?.oid}/artifacts/${typePath}`}/${
-        this.crawl?.id
-      }"
+      href="${this.baseUrl ||
+      `/orgs/${this.crawl?.oid}/artifacts/${typePath}`}/${this.crawl?.id}"
      @click=${async (e: MouseEvent) => {
        e.preventDefault();
        await this.updateComplete;
@ -521,15 +520,9 @@ export class CrawlList extends LitElement {
        <div class="col">
          <slot name="idCol">${msg("Name")}</slot>
        </div>
-        <div class="col">
-          ${this.artifactType === "upload" ? msg("Uploaded") : msg("Finished")}
-        </div>
+        <div class="col">${msg("Date Created")}</div>
        <div class="col">${msg("Size")}</div>
-        <div class="col">
-          ${this.artifactType === "upload"
-            ? msg("Uploaded By")
-            : msg("Started By")}
-        </div>
+        <div class="col">${msg("Created By")}</div>
        <div class="col action">
          <span class="srOnly">${msg("Actions")}</span>
        </div>
--- a/frontend/src/components/not-found.ts
+++ b/frontend/src/components/not-found.ts
@ -8,7 +8,9 @@ export class NotFound extends LitElement {
  }
  render() {
    return html`
-      <div class="text-xl text-gray-400">${msg("Page not found")}</div>
+      <div class="text-xl text-gray-400 text-center">
+        ${msg("Page not found")}
+      </div>
    `;
  }
 }
--- a/frontend/src/pages/org/collection-detail.ts
+++ b/frontend/src/pages/org/collection-detail.ts
@ -1,14 +1,25 @@
 import { state, property } from "lit/decorators.js";
 import { msg, localized, str } from "@lit/localize";
+import { choose } from "lit/directives/choose.js";
 import { when } from "lit/directives/when.js";
 import { guard } from "lit/directives/guard.js";
+import queryString from "query-string";

 import type { AuthState } from "../../utils/AuthService";
 import LiteElement, { html } from "../../utils/LiteElement";
 import type { Collection } from "../../types/collection";
-import type { IntersectEvent } from "../../components/observable";
+import type {
+  APIPaginatedList,
+  APIPaginationQuery,
+  APISortQuery,
+} from "../../types/api";
+import type { Crawl, CrawlState, Upload } from "../../types/crawler";
+import type { PageChangeEvent } from "../../components/pagination";

+const ABORT_REASON_THROTTLE = "throttled";
 const DESCRIPTION_MAX_HEIGHT_PX = 200;
+const TABS = ["replay", "web-captures"] as const;
+export type Tab = (typeof TABS)[number];

@localized()
 export class CollectionDetail extends LiteElement {
@ -21,26 +32,46 @@ export class CollectionDetail extends LiteElement {
  @property({ type: String })
  collectionId!: string;

+  @property({ type: String })
+  resourceTab?: Tab = TABS[0];
+
  @property({ type: Boolean })
  isCrawler?: boolean;

  @state()
  private collection?: Collection;

+  @state()
+  private webCaptures?: APIPaginatedList;
+
  @state()
  private openDialogName?: "delete";

-  @state()
-  private isDialogVisible: boolean = false;
-
  @state()
  private isDescriptionExpanded = false;

+  // Use to cancel requests
+  private getWebCapturesController: AbortController | null = null;
+
+  private readonly tabLabels: Record<Tab, { icon: any; text: string }> = {
+    replay: {
+      icon: { name: "link-replay", library: "app" },
+      text: msg("Replay"),
+    },
+    "web-captures": {
+      icon: { name: "list-ul", library: "default" },
+      text: msg("Web Captures"),
+    },
+  };
+
  protected async willUpdate(changedProperties: Map<string, any>) {
    if (changedProperties.has("orgId")) {
      this.collection = undefined;
      this.fetchCollection();
    }
+    if (changedProperties.has("collectionId")) {
+      this.fetchWebCaptures();
+    }
  }

  protected async updated(changedProperties: Map<string, any>) {
@ -51,25 +82,31 @@ export class CollectionDetail extends LiteElement {

  render() {
    return html`${this.renderHeader()}
-      <header class="md:flex justify-between items-end pb-3 border-b">
-        <h2
-          class="flex-1 min-w-0 text-xl font-semibold leading-10 truncate mr-2"
+      <header class="md:flex items-center gap-2 pb-3 mb-3 border-b">
+        <h1
+          class="flex-1 min-w-0 text-xl font-semibold leading-7 truncate mb-2 md:mb-0"
        >
-          ${this.collection?.name || html`<sl-skeleton></sl-skeleton>`}
-        </h2>
+          ${this.collection?.name ||
+          html`<sl-skeleton class="w-96"></sl-skeleton>`}
+        </h1>
        ${when(this.isCrawler, this.renderActions)}
      </header>
-      <div class="my-7">${this.renderDescription()}</div>
-      ${when(
-        this.collection?.resources.length,
-        () => html`<div>${this.renderReplay()}</div>`
+      <div class="mb-3">${this.renderTabs()}</div>
+
+      ${choose(
+        this.resourceTab,
+        [
+          ["replay", this.renderOverview],
+          ["web-captures", this.renderWebCaptures],
+        ],
+
+        () => html`<btrix-not-found></btrix-not-found>`
      )}

      <btrix-dialog
        label=${msg("Delete Collection?")}
        ?open=${this.openDialogName === "delete"}
        @sl-request-close=${() => (this.openDialogName = undefined)}
-        @sl-after-hide=${() => (this.isDialogVisible = false)}
      >
        ${msg(
          html`Are you sure you want to delete
@ -95,7 +132,7 @@ export class CollectionDetail extends LiteElement {
  }

  private renderHeader = () => html`
-    <nav class="mb-5">
+    <nav class="mb-7">
      <a
        class="text-gray-600 hover:text-gray-800 text-sm font-medium"
        href=${`/orgs/${this.orgId}/collections`}
@ -109,6 +146,31 @@ export class CollectionDetail extends LiteElement {
    </nav>
  `;

+  private renderTabs = () => {
+    return html`
+      <nav class="flex gap-2">
+        ${TABS.map((tabName) => {
+          const isSelected = tabName === this.resourceTab;
+          return html`
+            <btrix-button
+              variant=${isSelected ? "primary" : "neutral"}
+              ?raised=${isSelected}
+              aria-selected="${isSelected}"
+              href=${`/orgs/${this.orgId}/collections/view/${this.collectionId}/${tabName}`}
+              @click=${this.navLink}
+            >
+              <sl-icon
+                name=${this.tabLabels[tabName].icon.name}
+                library=${this.tabLabels[tabName].icon.library}
+              ></sl-icon>
+              ${this.tabLabels[tabName].text}</btrix-button
+            >
+          `;
+        })}
+      </nav>
+    `;
+  };
+
  private renderActions = () => {
    const authToken = this.authState!.headers.Authorization.split(" ")[1];

@ -157,9 +219,9 @@ export class CollectionDetail extends LiteElement {
    return html`
      <section>
        <header class="flex items-center justify-between">
-          <h3 class="text-lg font-semibold leading-none h-8 min-h-fit mb-1">
+          <h2 class="text-lg font-semibold leading-none h-8 min-h-fit mb-1">
            ${msg("Description")}
-          </h3>
+          </h2>
          ${when(
            this.isCrawler,
            () =>
@ -222,17 +284,97 @@ export class CollectionDetail extends LiteElement {
    `;
  }

+  private renderOverview = () => html`
+    ${this.renderReplay()}
+    <div class="my-7">${this.renderDescription()}</div>
+  `;
+
+  private renderWebCaptures = () => html`<section>
+    ${when(
+      this.webCaptures,
+      () => {
+        const { items, page, total, pageSize } = this.webCaptures!;
+        const hasItems = items.length;
+        return html`
+          <section>
+            ${hasItems ? this.renderWebCaptureList() : this.renderEmptyState()}
+          </section>
+          ${when(
+            hasItems || page > 1,
+            () => html`
+              <footer class="mt-6 flex justify-center">
+                <btrix-pagination
+                  page=${page}
+                  totalCount=${total}
+                  size=${pageSize}
+                  @page-change=${async (e: PageChangeEvent) => {
+                    await this.fetchWebCaptures({
+                      page: e.detail.page,
+                    });
+
+                    // Scroll to top of list
+                    // TODO once deep-linking is implemented, scroll to top of pushstate
+                    this.scrollIntoView({ behavior: "smooth" });
+                  }}
+                ></btrix-pagination>
+              </footer>
+            `
+          )}
+        `;
+      },
+      () => html`
+        <div class="w-full flex items-center justify-center my-12 text-2xl">
+          <sl-spinner></sl-spinner>
+        </div>
+      `
+    )}
+  </section>`;
+
+  private renderWebCaptureList() {
+    if (!this.webCaptures) return;
+
+    return html`
+      <btrix-crawl-list
+        baseUrl=${`/orgs/${this.orgId}/collections/view/${this.collectionId}/artifact`}
+      >
+        ${this.webCaptures.items.map(this.renderWebCaptureItem)}
+      </btrix-crawl-list>
+    `;
+  }
+
+  private renderEmptyState() {
+    if (this.webCaptures?.page && this.webCaptures?.page > 1) {
+      return html`
+        <div class="border-t border-b py-5">
+          <p class="text-center text-neutral-500">
+            ${msg("Could not find page.")}
+          </p>
+        </div>
+      `;
+    }
+
+    return html`
+      <div class="border-t border-b py-5">
+        <p class="text-center text-neutral-500">
+          ${msg("No matching web captures found.")}
+        </p>
+      </div>
+    `;
+  }
+
+  private renderWebCaptureItem = (wc: Crawl | Upload) =>
+    html`
+      <btrix-crawl-list-item .crawl=${wc}>
+        <div slot="menuTrigger" role="none"></div>
+      </btrix-crawl-list-item>
+    `;
+
  private renderReplay() {
    const replaySource = `/api/orgs/${this.orgId}/collections/${this.collectionId}/replay.json`;
    const headers = this.authState?.headers;
    const config = JSON.stringify({ headers });

    return html`<section>
-      <header class="flex items-center justify-between">
-        <h3 class="text-lg font-semibold leading-none h-8 min-h-fit mb-1">
-          ${msg("Replay")}
-        </h3>
-      </header>
      <main>
        <div class="aspect-4/3 border rounded-lg overflow-hidden">
          ${guard(
@ -332,5 +474,50 @@ export class CollectionDetail extends LiteElement {

    return data;
  }
+
+  /**
+   * Fetch web captures and update internal state
+   */
+  private async fetchWebCaptures(params?: APIPaginationQuery): Promise<void> {
+    this.cancelInProgressGetWebCaptures();
+    try {
+      this.webCaptures = await this.getWebCaptures();
+    } catch (e: any) {
+      if (e === ABORT_REASON_THROTTLE) {
+        console.debug("Fetch web captures aborted to throttle");
+      } else {
+        this.notify({
+          message: msg("Sorry, couldn't retrieve web captures at this time."),
+          variant: "danger",
+          icon: "exclamation-octagon",
+        });
+      }
+    }
+  }
+
+  private cancelInProgressGetWebCaptures() {
+    if (this.getWebCapturesController) {
+      this.getWebCapturesController.abort(ABORT_REASON_THROTTLE);
+      this.getWebCapturesController = null;
+    }
+  }
+
+  private async getWebCaptures(
+    params?: Partial<{
+      state: CrawlState[];
+    }> &
+      APIPaginationQuery &
+      APISortQuery
+  ): Promise<APIPaginatedList> {
+    const query = queryString.stringify(params || {}, {
+      arrayFormat: "comma",
+    });
+    const data: APIPaginatedList = await this.apiFetch(
+      `/orgs/${this.orgId}/all-crawls?collectionId=${this.collectionId}&${query}`,
+      this.authState!
+    );
+
+    return data;
+  }
 }
 customElements.define("btrix-collection-detail", CollectionDetail);
--- a/frontend/src/pages/org/collections-list.ts
+++ b/frontend/src/pages/org/collections-list.ts
@ -370,7 +370,7 @@ export class CollectionsList extends LiteElement {
            <div class="col-span-1 text-xs pl-3">${msg("Collection Name")}</div>
            <div class="col-span-1 text-xs">${msg("Top 3 Tags")}</div>
            <div class="col-span-1 text-xs">${msg("Last Updated")}</div>
-            <div class="col-span-1 text-xs">${msg("Total Crawls")}</div>
+            <div class="col-span-1 text-xs">${msg("Web Captures")}</div>
            <div class="col-span-2 text-xs">${msg("Total Pages")}</div>
          </div>
        </header>
@ -447,6 +447,7 @@ export class CollectionsList extends LiteElement {
            <a
              href=${`/orgs/${this.orgId}/collections/view/${col.id}`}
              class="block text-primary hover:text-indigo-500"
+              @click=${this.navLink}
            >
              ${col.name}
            </a>
@ -473,8 +474,10 @@ export class CollectionsList extends LiteElement {
            class="col-span-1 truncate text-xs text-neutral-500 font-monostyle"
          >
            ${col.crawlCount === 1
-              ? msg("1 crawl")
-              : msg(str`${this.numberFormatter.format(col.crawlCount)} crawls`)}
+              ? msg("1 capture")
+              : msg(
+                  str`${this.numberFormatter.format(col.crawlCount)} captures`
+                )}
          </div>
          <div
            class="col-span-1 truncate text-xs text-neutral-500 font-monostyle"
--- a/frontend/src/pages/org/collections-new.ts
+++ b/frontend/src/pages/org/collections-new.ts
@ -40,7 +40,7 @@ export class CollectionsNew extends LiteElement {
  }

  private renderHeader = () => html`
-    <nav class="mb-5">
+    <nav class="mb-7">
      <a
        class="text-gray-600 hover:text-gray-800 text-sm font-medium"
        href=${`/orgs/${this.orgId}/collections`}
@ -69,9 +69,7 @@ export class CollectionsNew extends LiteElement {
      );

      this.notify({
-        message: msg(
-          str`Successfully created "${data.name}" Collection.`
-        ),
+        message: msg(str`Successfully created "${data.name}" Collection.`),
        variant: "success",
        icon: "check2-circle",
        duration: 8000,
--- a/frontend/src/pages/org/crawl-detail.ts
+++ b/frontend/src/pages/org/crawl-detail.ts
@ -230,32 +230,37 @@ export class CrawlDetail extends LiteElement {

    // TODO abstract into breadcrumbs
    const isWorkflowArtifact = this.crawlsBaseUrl.includes("/workflows/");
+    const isCollectionArtifact = this.crawlsBaseUrl.includes("/collections/");
+
+    let label = msg("Back to All Crawls");
+    if (isWorkflowArtifact) {
+      label = msg("Back to Crawl Workflow");
+    } else if (isCollectionArtifact) {
+      label = msg("Back to Collection");
+    } else if (this.crawl?.type === "upload") {
+      label = msg("Back to All Uploads");
+    }

    return html`
      <div class="mb-7">
        <a
          class="text-neutral-500 hover:text-neutral-600 text-sm font-medium"
-          href="${this.crawlsBaseUrl}?artifactType=${this.crawl?.type}"
+          href="${this.crawlsBaseUrl}${isWorkflowArtifact ||
+          isCollectionArtifact
+            ? ""
+            : `?artifactType=${this.crawl?.type}`}"
          @click=${this.navLink}
        >
          <sl-icon
            name="arrow-left"
            class="inline-block align-middle"
          ></sl-icon>
-          <span class="inline-block align-middle"
-            >${isWorkflowArtifact
-              ? msg("Back to Crawl Workflow")
-              : this.crawl?.type === "upload"
-              ? msg("Back to All Uploads")
-              : msg("Back to All Crawls")}</span
-          >
+          <span class="inline-block align-middle">${label}</span>
        </a>
      </div>

      <div class="mb-4">${this.renderHeader()}</div>

-      <hr class="mb-4" />
-
      <main>
        <section class="grid grid-cols-6 gap-4">
          <div class="col-span-6 md:col-span-1">${this.renderNav()}</div>
@ -384,8 +389,10 @@ export class CrawlDetail extends LiteElement {

  private renderHeader() {
    return html`
-      <header class="md:flex justify-between items-end">
-        <h1 class="text-xl font-semibold mb-4 md:mb-0 md:mr-2">
+      <header class="md:flex items-center gap-2 pb-3 mb-3 border-b">
+        <h1
+          class="flex-1 min-w-0 text-xl font-semibold leading-7 truncate mb-2 md:mb-0"
+        >
          ${this.renderName()}
        </h1>
        <div
--- a/frontend/src/pages/org/crawls-list.ts
+++ b/frontend/src/pages/org/crawls-list.ts
@ -249,8 +249,10 @@ export class CrawlsList extends LiteElement {
    return html`
      <main>
        <header class="contents">
-          <div class="flex justify-between w-full pb-4 mb-3 border-b">
-            <h1 class="text-xl font-semibold h-8">
+          <div class="md:flex items-center gap-2 pb-3 mb-3 border-b">
+            <h1
+              class="flex-1 min-w-0 text-xl font-semibold leading-7 truncate mb-2 md:mb-0"
+            >
              ${msg("All Archived Data")}
            </h1>
            ${when(
--- a/frontend/src/pages/org/index.ts
+++ b/frontend/src/pages/org/index.ts
@ -30,6 +30,7 @@ import type {
  UserRoleChangeEvent,
  OrgRemoveMemberEvent,
 } from "./settings";
+import type { Tab as CollectionTab } from "./collection-detail";

 export type OrgTab =
  | "crawls"
@ -45,6 +46,7 @@ type Params = {
  browserId?: string;
  artifactId?: string;
  resourceId?: string;
+  resourceTab?: string;
  artifactType?: Crawl["type"];
 };

@ -238,7 +240,9 @@ export class Org extends LiteElement {
    const crawlsAPIBaseUrl = `/orgs/${this.orgId}/crawls`;
    const crawlsBaseUrl = `/orgs/${this.orgId}/artifacts/crawls`;

-    const artifactType = this.orgPath.includes("/artifacts/upload") ? "upload" : "crawl";
+    const artifactType = this.orgPath.includes("/artifacts/upload")
+      ? "upload"
+      : "crawl";

    if (this.params.crawlOrWorkflowId) {
      return html` <btrix-crawl-detail
@ -345,34 +349,43 @@ export class Org extends LiteElement {
  private renderCollections() {
    if (this.params.resourceId) {
      if (this.orgPath.includes(`/edit/${this.params.resourceId}`)) {
-        return html`<div class="lg:px-5">
-          <btrix-collection-edit
-            .authState=${this.authState!}
-            orgId=${this.orgId!}
-            collectionId=${this.params.resourceId}
-            ?isCrawler=${this.isCrawler}
-          ></btrix-collection-edit>
-        </div>`;
-      }
-
-      return html`<div class="lg:px-5">
-        <btrix-collection-detail
+        return html`<btrix-collection-edit
          .authState=${this.authState!}
          orgId=${this.orgId!}
          collectionId=${this.params.resourceId}
          ?isCrawler=${this.isCrawler}
-        ></btrix-collection-detail>
-      </div>`;
+        ></btrix-collection-edit>`;
+      }
+
+      if (this.params.artifactId) {
+        const crawlsAPIBaseUrl = `/orgs/${this.orgId}/crawls`;
+        // TODO abstract into breadcrumbs
+        const crawlsBaseUrl = `/orgs/${this.orgId}/collections/view/${this.params.resourceId}/web-captures`;
+
+        return html` <btrix-crawl-detail
+          .authState=${this.authState!}
+          crawlId=${this.params.artifactId}
+          crawlsAPIBaseUrl=${crawlsAPIBaseUrl}
+          crawlsBaseUrl=${crawlsBaseUrl}
+          ?isCrawler=${this.isCrawler}
+        ></btrix-crawl-detail>`;
+      }
+
+      return html`<btrix-collection-detail
+        .authState=${this.authState!}
+        orgId=${this.orgId!}
+        collectionId=${this.params.resourceId}
+        resourceTab=${(this.params.resourceTab as CollectionTab) || "replay"}
+        ?isCrawler=${this.isCrawler}
+      ></btrix-collection-detail>`;
    }

    if (this.orgPath.endsWith("/new")) {
-      return html`<div class="lg:px-5">
-        <btrix-collections-new
-          .authState=${this.authState!}
-          orgId=${this.orgId!}
-          ?isCrawler=${this.isCrawler}
-        ></btrix-collections-new>
-      </div>`;
+      return html`<btrix-collections-new
+        .authState=${this.authState!}
+        orgId=${this.orgId!}
+        ?isCrawler=${this.isCrawler}
+      ></btrix-collections-new>`;
    }

    return html`<btrix-collections-list
--- a/frontend/src/routes.ts
+++ b/frontend/src/routes.ts
@ -14,7 +14,7 @@ export const ROUTES = {
    "/orgs/:orgId/:orgTab",
    // Optional segments:
    "(/new)",
-    "(/view/:resourceId)",
+    "(/view/:resourceId(/:resourceTab))",
    "(/edit/:resourceId)",
    "(/crawls)",
    "(/crawl/:crawlOrWorkflowId)",