List web captures in Collection (#1024)

- Adds tab for "Web Captures" in Collection detail view
- Move Collection description under Replay section
- Fixes app reloading when clicking into a Collection
- Standardizes Web Capture list headers from "Finished -> "Created Date"

frontend/src/components/button.ts

@@ -129,6 +129,7 @@ export class Button extends LitElement {
         raised: this.raised,
       })}
       ?disabled=${this.disabled}
+      href=${ifDefined(this.href)}
       aria-label=${ifDefined(this.label)}
       @click=${this.handleClick}
     >

frontend/src/components/crawl-list.ts

@@ -226,9 +226,8 @@ export class CrawlListItem extends LitElement {
     return html`<a
       class="item row"
       role="button"
-      href="${this.baseUrl || `/orgs/${this.crawl?.oid}/artifacts/${typePath}`}/${
+      href="${this.baseUrl ||
-        this.crawl?.id
+      `/orgs/${this.crawl?.oid}/artifacts/${typePath}`}/${this.crawl?.id}"
-      }"
       @click=${async (e: MouseEvent) => {
         e.preventDefault();
         await this.updateComplete;
@@ -521,15 +520,9 @@ export class CrawlList extends LitElement {
         <div class="col">
           <slot name="idCol">${msg("Name")}</slot>
         </div>
-        <div class="col">
+        <div class="col">${msg("Date Created")}</div>
-          ${this.artifactType === "upload" ? msg("Uploaded") : msg("Finished")}
-        </div>
         <div class="col">${msg("Size")}</div>
-        <div class="col">
+        <div class="col">${msg("Created By")}</div>
-          ${this.artifactType === "upload"
-            ? msg("Uploaded By")
-            : msg("Started By")}
-        </div>
         <div class="col action">
           <span class="srOnly">${msg("Actions")}</span>
         </div>

frontend/src/components/not-found.ts

@@ -8,7 +8,9 @@ export class NotFound extends LitElement {
   }
   render() {
     return html`
-      <div class="text-xl text-gray-400">${msg("Page not found")}</div>
+      <div class="text-xl text-gray-400 text-center">
+        ${msg("Page not found")}
+      </div>
     `;
   }
 }

frontend/src/pages/org/collection-detail.ts

@@ -1,14 +1,25 @@
 import { state, property } from "lit/decorators.js";
 import { msg, localized, str } from "@lit/localize";
+import { choose } from "lit/directives/choose.js";
 import { when } from "lit/directives/when.js";
 import { guard } from "lit/directives/guard.js";
+import queryString from "query-string";
 
 import type { AuthState } from "../../utils/AuthService";
 import LiteElement, { html } from "../../utils/LiteElement";
 import type { Collection } from "../../types/collection";
-import type { IntersectEvent } from "../../components/observable";
+import type {
+  APIPaginatedList,
+  APIPaginationQuery,
+  APISortQuery,
+} from "../../types/api";
+import type { Crawl, CrawlState, Upload } from "../../types/crawler";
+import type { PageChangeEvent } from "../../components/pagination";
 
+const ABORT_REASON_THROTTLE = "throttled";
 const DESCRIPTION_MAX_HEIGHT_PX = 200;
+const TABS = ["replay", "web-captures"] as const;
+export type Tab = (typeof TABS)[number];
 
 @localized()
 export class CollectionDetail extends LiteElement {
@@ -21,26 +32,46 @@ export class CollectionDetail extends LiteElement {
   @property({ type: String })
   collectionId!: string;
 
+  @property({ type: String })
+  resourceTab?: Tab = TABS[0];
+
   @property({ type: Boolean })
   isCrawler?: boolean;
 
   @state()
   private collection?: Collection;
 
+  @state()
+  private webCaptures?: APIPaginatedList;
+
   @state()
   private openDialogName?: "delete";
 
-  @state()
-  private isDialogVisible: boolean = false;
-
   @state()
   private isDescriptionExpanded = false;
 
+  // Use to cancel requests
+  private getWebCapturesController: AbortController | null = null;
+
+  private readonly tabLabels: Record<Tab, { icon: any; text: string }> = {
+    replay: {
+      icon: { name: "link-replay", library: "app" },
+      text: msg("Replay"),
+    },
+    "web-captures": {
+      icon: { name: "list-ul", library: "default" },
+      text: msg("Web Captures"),
+    },
+  };
+
   protected async willUpdate(changedProperties: Map<string, any>) {
     if (changedProperties.has("orgId")) {
       this.collection = undefined;
       this.fetchCollection();
     }
+    if (changedProperties.has("collectionId")) {
+      this.fetchWebCaptures();
+    }
   }
 
   protected async updated(changedProperties: Map<string, any>) {
@@ -51,25 +82,31 @@ export class CollectionDetail extends LiteElement {
 
   render() {
     return html`${this.renderHeader()}
-      <header class="md:flex justify-between items-end pb-3 border-b">
+      <header class="md:flex items-center gap-2 pb-3 mb-3 border-b">
-        <h2
+        <h1
-          class="flex-1 min-w-0 text-xl font-semibold leading-10 truncate mr-2"
+          class="flex-1 min-w-0 text-xl font-semibold leading-7 truncate mb-2 md:mb-0"
         >
-          ${this.collection?.name || html`<sl-skeleton></sl-skeleton>`}
+          ${this.collection?.name ||
-        </h2>
+          html`<sl-skeleton class="w-96"></sl-skeleton>`}
+        </h1>
         ${when(this.isCrawler, this.renderActions)}
       </header>
-      <div class="my-7">${this.renderDescription()}</div>
+      <div class="mb-3">${this.renderTabs()}</div>
-      ${when(
+
-        this.collection?.resources.length,
+      ${choose(
-        () => html`<div>${this.renderReplay()}</div>`
+        this.resourceTab,
+        [
+          ["replay", this.renderOverview],
+          ["web-captures", this.renderWebCaptures],
+        ],
+
+        () => html`<btrix-not-found></btrix-not-found>`
       )}
 
       <btrix-dialog
         label=${msg("Delete Collection?")}
         ?open=${this.openDialogName === "delete"}
         @sl-request-close=${() => (this.openDialogName = undefined)}
-        @sl-after-hide=${() => (this.isDialogVisible = false)}
       >
         ${msg(
           html`Are you sure you want to delete
@@ -95,7 +132,7 @@ export class CollectionDetail extends LiteElement {
   }
 
   private renderHeader = () => html`
-    <nav class="mb-5">
+    <nav class="mb-7">
       <a
         class="text-gray-600 hover:text-gray-800 text-sm font-medium"
         href=${`/orgs/${this.orgId}/collections`}
@@ -109,6 +146,31 @@ export class CollectionDetail extends LiteElement {
     </nav>
   `;
 
+  private renderTabs = () => {
+    return html`
+      <nav class="flex gap-2">
+        ${TABS.map((tabName) => {
+          const isSelected = tabName === this.resourceTab;
+          return html`
+            <btrix-button
+              variant=${isSelected ? "primary" : "neutral"}
+              ?raised=${isSelected}
+              aria-selected="${isSelected}"
+              href=${`/orgs/${this.orgId}/collections/view/${this.collectionId}/${tabName}`}
+              @click=${this.navLink}
+            >
+              <sl-icon
+                name=${this.tabLabels[tabName].icon.name}
+                library=${this.tabLabels[tabName].icon.library}
+              ></sl-icon>
+              ${this.tabLabels[tabName].text}</btrix-button
+            >
+          `;
+        })}
+      </nav>
+    `;
+  };
+
   private renderActions = () => {
     const authToken = this.authState!.headers.Authorization.split(" ")[1];
 
@@ -157,9 +219,9 @@ export class CollectionDetail extends LiteElement {
     return html`
       <section>
         <header class="flex items-center justify-between">
-          <h3 class="text-lg font-semibold leading-none h-8 min-h-fit mb-1">
+          <h2 class="text-lg font-semibold leading-none h-8 min-h-fit mb-1">
             ${msg("Description")}
-          </h3>
+          </h2>
           ${when(
             this.isCrawler,
             () =>
@@ -222,17 +284,97 @@ export class CollectionDetail extends LiteElement {
     `;
   }
 
+  private renderOverview = () => html`
+    ${this.renderReplay()}
+    <div class="my-7">${this.renderDescription()}</div>
+  `;
+
+  private renderWebCaptures = () => html`<section>
+    ${when(
+      this.webCaptures,
+      () => {
+        const { items, page, total, pageSize } = this.webCaptures!;
+        const hasItems = items.length;
+        return html`
+          <section>
+            ${hasItems ? this.renderWebCaptureList() : this.renderEmptyState()}
+          </section>
+          ${when(
+            hasItems || page > 1,
+            () => html`
+              <footer class="mt-6 flex justify-center">
+                <btrix-pagination
+                  page=${page}
+                  totalCount=${total}
+                  size=${pageSize}
+                  @page-change=${async (e: PageChangeEvent) => {
+                    await this.fetchWebCaptures({
+                      page: e.detail.page,
+                    });
+
+                    // Scroll to top of list
+                    // TODO once deep-linking is implemented, scroll to top of pushstate
+                    this.scrollIntoView({ behavior: "smooth" });
+                  }}
+                ></btrix-pagination>
+              </footer>
+            `
+          )}
+        `;
+      },
+      () => html`
+        <div class="w-full flex items-center justify-center my-12 text-2xl">
+          <sl-spinner></sl-spinner>
+        </div>
+      `
+    )}
+  </section>`;
+
+  private renderWebCaptureList() {
+    if (!this.webCaptures) return;
+
+    return html`
+      <btrix-crawl-list
+        baseUrl=${`/orgs/${this.orgId}/collections/view/${this.collectionId}/artifact`}
+      >
+        ${this.webCaptures.items.map(this.renderWebCaptureItem)}
+      </btrix-crawl-list>
+    `;
+  }
+
+  private renderEmptyState() {
+    if (this.webCaptures?.page && this.webCaptures?.page > 1) {
+      return html`
+        <div class="border-t border-b py-5">
+          <p class="text-center text-neutral-500">
+            ${msg("Could not find page.")}
+          </p>
+        </div>
+      `;
+    }
+
+    return html`
+      <div class="border-t border-b py-5">
+        <p class="text-center text-neutral-500">
+          ${msg("No matching web captures found.")}
+        </p>
+      </div>
+    `;
+  }
+
+  private renderWebCaptureItem = (wc: Crawl | Upload) =>
+    html`
+      <btrix-crawl-list-item .crawl=${wc}>
+        <div slot="menuTrigger" role="none"></div>
+      </btrix-crawl-list-item>
+    `;
+
   private renderReplay() {
     const replaySource = `/api/orgs/${this.orgId}/collections/${this.collectionId}/replay.json`;
     const headers = this.authState?.headers;
     const config = JSON.stringify({ headers });
 
     return html`<section>
-      <header class="flex items-center justify-between">
-        <h3 class="text-lg font-semibold leading-none h-8 min-h-fit mb-1">
-          ${msg("Replay")}
-        </h3>
-      </header>
       <main>
         <div class="aspect-4/3 border rounded-lg overflow-hidden">
           ${guard(
@@ -332,5 +474,50 @@ export class CollectionDetail extends LiteElement {
 
     return data;
   }
+
+  /**
+   * Fetch web captures and update internal state
+   */
+  private async fetchWebCaptures(params?: APIPaginationQuery): Promise<void> {
+    this.cancelInProgressGetWebCaptures();
+    try {
+      this.webCaptures = await this.getWebCaptures();
+    } catch (e: any) {
+      if (e === ABORT_REASON_THROTTLE) {
+        console.debug("Fetch web captures aborted to throttle");
+      } else {
+        this.notify({
+          message: msg("Sorry, couldn't retrieve web captures at this time."),
+          variant: "danger",
+          icon: "exclamation-octagon",
+        });
+      }
+    }
+  }
+
+  private cancelInProgressGetWebCaptures() {
+    if (this.getWebCapturesController) {
+      this.getWebCapturesController.abort(ABORT_REASON_THROTTLE);
+      this.getWebCapturesController = null;
+    }
+  }
+
+  private async getWebCaptures(
+    params?: Partial<{
+      state: CrawlState[];
+    }> &
+      APIPaginationQuery &
+      APISortQuery
+  ): Promise<APIPaginatedList> {
+    const query = queryString.stringify(params || {}, {
+      arrayFormat: "comma",
+    });
+    const data: APIPaginatedList = await this.apiFetch(
+      `/orgs/${this.orgId}/all-crawls?collectionId=${this.collectionId}&${query}`,
+      this.authState!
+    );
+
+    return data;
+  }
 }
 customElements.define("btrix-collection-detail", CollectionDetail);

frontend/src/pages/org/collections-list.ts

@@ -370,7 +370,7 @@ export class CollectionsList extends LiteElement {
             <div class="col-span-1 text-xs pl-3">${msg("Collection Name")}</div>
             <div class="col-span-1 text-xs">${msg("Top 3 Tags")}</div>
             <div class="col-span-1 text-xs">${msg("Last Updated")}</div>
-            <div class="col-span-1 text-xs">${msg("Total Crawls")}</div>
+            <div class="col-span-1 text-xs">${msg("Web Captures")}</div>
             <div class="col-span-2 text-xs">${msg("Total Pages")}</div>
           </div>
         </header>
@@ -447,6 +447,7 @@ export class CollectionsList extends LiteElement {
             <a
               href=${`/orgs/${this.orgId}/collections/view/${col.id}`}
               class="block text-primary hover:text-indigo-500"
+              @click=${this.navLink}
             >
               ${col.name}
             </a>
@@ -473,8 +474,10 @@ export class CollectionsList extends LiteElement {
             class="col-span-1 truncate text-xs text-neutral-500 font-monostyle"
           >
             ${col.crawlCount === 1
-              ? msg("1 crawl")
+              ? msg("1 capture")
-              : msg(str`${this.numberFormatter.format(col.crawlCount)} crawls`)}
+              : msg(
+                  str`${this.numberFormatter.format(col.crawlCount)} captures`
+                )}
           </div>
           <div
             class="col-span-1 truncate text-xs text-neutral-500 font-monostyle"

frontend/src/pages/org/collections-new.ts

@@ -40,7 +40,7 @@ export class CollectionsNew extends LiteElement {
   }
 
   private renderHeader = () => html`
-    <nav class="mb-5">
+    <nav class="mb-7">
       <a
         class="text-gray-600 hover:text-gray-800 text-sm font-medium"
         href=${`/orgs/${this.orgId}/collections`}
@@ -69,9 +69,7 @@ export class CollectionsNew extends LiteElement {
       );
 
       this.notify({
-        message: msg(
+        message: msg(str`Successfully created "${data.name}" Collection.`),
-          str`Successfully created "${data.name}" Collection.`
-        ),
         variant: "success",
         icon: "check2-circle",
         duration: 8000,

frontend/src/pages/org/crawl-detail.ts

@@ -230,32 +230,37 @@ export class CrawlDetail extends LiteElement {
 
     // TODO abstract into breadcrumbs
     const isWorkflowArtifact = this.crawlsBaseUrl.includes("/workflows/");
+    const isCollectionArtifact = this.crawlsBaseUrl.includes("/collections/");
+
+    let label = msg("Back to All Crawls");
+    if (isWorkflowArtifact) {
+      label = msg("Back to Crawl Workflow");
+    } else if (isCollectionArtifact) {
+      label = msg("Back to Collection");
+    } else if (this.crawl?.type === "upload") {
+      label = msg("Back to All Uploads");
+    }
 
     return html`
       <div class="mb-7">
         <a
           class="text-neutral-500 hover:text-neutral-600 text-sm font-medium"
-          href="${this.crawlsBaseUrl}?artifactType=${this.crawl?.type}"
+          href="${this.crawlsBaseUrl}${isWorkflowArtifact ||
+          isCollectionArtifact
+            ? ""
+            : `?artifactType=${this.crawl?.type}`}"
           @click=${this.navLink}
         >
           <sl-icon
             name="arrow-left"
             class="inline-block align-middle"
           ></sl-icon>
-          <span class="inline-block align-middle"
+          <span class="inline-block align-middle">${label}</span>
-            >${isWorkflowArtifact
-              ? msg("Back to Crawl Workflow")
-              : this.crawl?.type === "upload"
-              ? msg("Back to All Uploads")
-              : msg("Back to All Crawls")}</span
-          >
         </a>
       </div>
 
       <div class="mb-4">${this.renderHeader()}</div>
 
-      <hr class="mb-4" />
-
       <main>
         <section class="grid grid-cols-6 gap-4">
           <div class="col-span-6 md:col-span-1">${this.renderNav()}</div>
@@ -384,8 +389,10 @@ export class CrawlDetail extends LiteElement {
 
   private renderHeader() {
     return html`
-      <header class="md:flex justify-between items-end">
+      <header class="md:flex items-center gap-2 pb-3 mb-3 border-b">
-        <h1 class="text-xl font-semibold mb-4 md:mb-0 md:mr-2">
+        <h1
+          class="flex-1 min-w-0 text-xl font-semibold leading-7 truncate mb-2 md:mb-0"
+        >
           ${this.renderName()}
         </h1>
         <div

frontend/src/pages/org/crawls-list.ts

@@ -249,8 +249,10 @@ export class CrawlsList extends LiteElement {
     return html`
       <main>
         <header class="contents">
-          <div class="flex justify-between w-full pb-4 mb-3 border-b">
+          <div class="md:flex items-center gap-2 pb-3 mb-3 border-b">
-            <h1 class="text-xl font-semibold h-8">
+            <h1
+              class="flex-1 min-w-0 text-xl font-semibold leading-7 truncate mb-2 md:mb-0"
+            >
               ${msg("All Archived Data")}
             </h1>
             ${when(

frontend/src/pages/org/index.ts

@@ -30,6 +30,7 @@ import type {
   UserRoleChangeEvent,
   OrgRemoveMemberEvent,
 } from "./settings";
+import type { Tab as CollectionTab } from "./collection-detail";
 
 export type OrgTab =
   | "crawls"
@@ -45,6 +46,7 @@ type Params = {
   browserId?: string;
   artifactId?: string;
   resourceId?: string;
+  resourceTab?: string;
   artifactType?: Crawl["type"];
 };
 
@@ -238,7 +240,9 @@ export class Org extends LiteElement {
     const crawlsAPIBaseUrl = `/orgs/${this.orgId}/crawls`;
     const crawlsBaseUrl = `/orgs/${this.orgId}/artifacts/crawls`;
 
-    const artifactType = this.orgPath.includes("/artifacts/upload") ? "upload" : "crawl";
+    const artifactType = this.orgPath.includes("/artifacts/upload")
+      ? "upload"
+      : "crawl";
 
     if (this.params.crawlOrWorkflowId) {
       return html` <btrix-crawl-detail
@@ -345,34 +349,43 @@ export class Org extends LiteElement {
   private renderCollections() {
     if (this.params.resourceId) {
       if (this.orgPath.includes(`/edit/${this.params.resourceId}`)) {
-        return html`<div class="lg:px-5">
+        return html`<btrix-collection-edit
-          <btrix-collection-edit
-            .authState=${this.authState!}
-            orgId=${this.orgId!}
-            collectionId=${this.params.resourceId}
-            ?isCrawler=${this.isCrawler}
-          ></btrix-collection-edit>
-        </div>`;
-      }
-
-      return html`<div class="lg:px-5">
-        <btrix-collection-detail
           .authState=${this.authState!}
           orgId=${this.orgId!}
           collectionId=${this.params.resourceId}
           ?isCrawler=${this.isCrawler}
-        ></btrix-collection-detail>
+        ></btrix-collection-edit>`;
-      </div>`;
+      }
+
+      if (this.params.artifactId) {
+        const crawlsAPIBaseUrl = `/orgs/${this.orgId}/crawls`;
+        // TODO abstract into breadcrumbs
+        const crawlsBaseUrl = `/orgs/${this.orgId}/collections/view/${this.params.resourceId}/web-captures`;
+
+        return html` <btrix-crawl-detail
+          .authState=${this.authState!}
+          crawlId=${this.params.artifactId}
+          crawlsAPIBaseUrl=${crawlsAPIBaseUrl}
+          crawlsBaseUrl=${crawlsBaseUrl}
+          ?isCrawler=${this.isCrawler}
+        ></btrix-crawl-detail>`;
+      }
+
+      return html`<btrix-collection-detail
+        .authState=${this.authState!}
+        orgId=${this.orgId!}
+        collectionId=${this.params.resourceId}
+        resourceTab=${(this.params.resourceTab as CollectionTab) || "replay"}
+        ?isCrawler=${this.isCrawler}
+      ></btrix-collection-detail>`;
     }
 
     if (this.orgPath.endsWith("/new")) {
-      return html`<div class="lg:px-5">
+      return html`<btrix-collections-new
-        <btrix-collections-new
+        .authState=${this.authState!}
-          .authState=${this.authState!}
+        orgId=${this.orgId!}
-          orgId=${this.orgId!}
+        ?isCrawler=${this.isCrawler}
-          ?isCrawler=${this.isCrawler}
+      ></btrix-collections-new>`;
-        ></btrix-collections-new>
-      </div>`;
     }
 
     return html`<btrix-collections-list

frontend/src/routes.ts

@@ -14,7 +14,7 @@ export const ROUTES = {
     "/orgs/:orgId/:orgTab",
     // Optional segments:
     "(/new)",
-    "(/view/:resourceId)",
+    "(/view/:resourceId(/:resourceTab))",
     "(/edit/:resourceId)",
     "(/crawls)",
     "(/crawl/:crawlOrWorkflowId)",