Show critical errors in Crawl detail logs (#811)

2023-05-05 11:30:38 -07:00 · 2023-05-05 11:30:38 -07:00 · 85c96de883
commit 85c96de883
parent 53539425aa
8 changed files with 400 additions and 149 deletions
--- a/frontend/src/components/crawl-logs.ts
+++ b/frontend/src/components/crawl-logs.ts
@ -0,0 +1,118 @@
+import { LitElement, html, css } from "lit";
+import { property, state } from "lit/decorators.js";
+import { when } from "lit/directives/when.js";
+import { msg, localized, str } from "@lit/localize";
+
+import { truncate } from "../utils/css";
+import type { APIPaginatedList } from "../types/api";
+
+type CrawlLog = {
+  timestamp: string;
+  logLevel: "error";
+  details: Record<string, any>;
+  context: string;
+  message: string;
+};
+
+@localized()
+export class CrawlLogs extends LitElement {
+  static styles = [
+    truncate,
+    css`
+      btrix-numbered-list {
+        font-size: var(--sl-font-size-x-small);
+      }
+
+      .row {
+        display: grid;
+        grid-template-columns: 9rem 4rem 14rem 1fr;
+        line-height: 1.3;
+        max-width: 800px;
+      }
+
+      .cell {
+        padding-left: var(--sl-spacing-x-small);
+        padding-right: var(--sl-spacing-x-small);
+      }
+
+      .tag {
+        display: inline-block;
+        border-radius: var(--sl-border-radius-small);
+        padding: var(--sl-spacing-3x-small) var(--sl-spacing-2x-small);
+        text-transform: capitalize;
+        /* TODO handle non-errors */
+        background-color: var(--danger);
+        color: var(--sl-color-neutral-0);
+      }
+
+      footer {
+        display: flex;
+        justify-content: center;
+        margin-top: var(--sl-spacing-large);
+        margin-bottom: var(--sl-spacing-x-large);
+      }
+
+      .message {
+        white-space: pre-wrap;
+      }
+
+      .url {
+        text-overflow: ellipsis;
+        overflow: hidden;
+        white-space: nowrap;
+      }
+
+    `,
+  ];
+
+  @property({ type: Object })
+  logs?: APIPaginatedList;
+
+  render() {
+    if (!this.logs) return;
+    return html`<btrix-numbered-list>
+        <btrix-numbered-list-header slot="header">
+          <div class="row">
+            <div class="cell">${msg("Date")}</div>
+            <div class="cell">${msg("Level")}</div>
+            <div class="cell">${msg("Error Message")}</div>
+            <div class="cell">${msg("Page URL")}</div>
+          </div>
+        </btrix-numbered-list-header>
+        ${this.logs.items.map(
+          (log: CrawlLog, idx) => html`
+            <btrix-numbered-list-item>
+              <div class="row">
+                <div>
+                  <sl-format-date
+                    date=${log.timestamp}
+                    month="2-digit"
+                    day="2-digit"
+                    year="2-digit"
+                    hour="2-digit"
+                    minute="2-digit"
+                    second="2-digit"
+                    hour-format="24"
+                  >
+                  </sl-format-date>
+                </div>
+                <div>
+                  <span class="tag">${log.logLevel}</span>
+                </div>
+                <div class="message">${log.message}</div>
+                <div class="url" title="${log.details?.page}"><a target="_blank" href="${log.details?.page}">${log.details?.page}</a></div>
+              </div>
+            </btrix-numbered-list-item>
+          `
+        )}
+      </btrix-numbered-list>
+      <footer>
+        <btrix-pagination
+          page=${this.logs.page}
+          totalCount=${this.logs.total}
+          size=${this.logs.pageSize}
+        >
+        </btrix-pagination>
+      </footer> `;
+  }
+}
--- a/frontend/src/components/crawl-pending-exclusions.ts
+++ b/frontend/src/components/crawl-pending-exclusions.ts
@ -87,20 +87,24 @@ export class CrawlPendingExclusions extends LiteElement {
    }

    return html`
-      <btrix-numbered-list
-        class="text-xs break-all"
-        .items=${this.pageResults.map((url, idx) => ({
-          order: idx + 1 + (this.page - 1) * this.pageSize,
-          content: html`<a
-            href=${url}
-            target="_blank"
-            rel="noopener noreferrer nofollow"
-            >${url}</a
-          >`,
-        }))}
-        aria-live="polite"
-        style="--link-color: var(--sl-color-danger-600); --link-hover-color: var(--sl-color-danger-400);"
-      ></btrix-numbered-list>
+      <btrix-numbered-list class="text-xs break-all" aria-live="polite">
+        ${this.pageResults.map(
+          (url, idx) => html`
+            <btrix-numbered-list-item>
+              <span class="text-red-600" slot="marker"
+                >${idx + 1 + (this.page - 1) * this.pageSize}.</span
+              >
+              <a
+                class="text-red-600 hover:text-red-500"
+                href=${url}
+                target="_blank"
+                rel="noopener noreferrer nofollow"
+                >${url}</a
+              >
+            </btrix-numbered-list-item>
+          `
+        )}
+      </btrix-numbered-list>
    `;
  }
 }
--- a/frontend/src/components/crawl-queue.ts
+++ b/frontend/src/components/crawl-queue.ts
@ -100,29 +100,28 @@ export class CrawlQueue extends LiteElement {

    if (!this.queue) return;

-    const excludedURLStyles = [
-      "--marker-color: var(--sl-color-danger-500)",
-      "--link-color: var(--sl-color-danger-500)",
-      "--link-hover-color: var(--sl-color-danger-400)",
-    ].join(";");
-
    return html`
-      <btrix-numbered-list
-        class="text-xs break-all"
-        .items=${this.queue.results.map((url, idx) => ({
-          order: idx + 1,
-          style: this.queue!.matched.some((v) => v === url)
-            ? excludedURLStyles
-            : "",
-          content: html`<a
-            href=${url}
-            target="_blank"
-            rel="noopener noreferrer nofollow"
-            >${url}</a
-          >`,
-        }))}
-        aria-live="polite"
-      ></btrix-numbered-list>
+      <btrix-numbered-list class="text-xs break-all" aria-live="polite">
+        ${this.queue.results.map((url, idx) => {
+          const isMatch = this.queue!.matched.some((v) => v === url);
+          return html`
+            <btrix-numbered-list-item>
+              <span class="${isMatch ? "text-red-600" : ""}" slot="marker"
+                >${idx + 1}.</span
+              >
+              <a
+                class="${isMatch
+                  ? "text-red-500 hover:text-red-400"
+                  : "text-blue-500 hover:text-blue-400"}"
+                href=${url}
+                target="_blank"
+                rel="noopener noreferrer nofollow"
+                >${url}</a
+              >
+            </btrix-numbered-list-item>
+          `;
+        })}
+      </btrix-numbered-list>

      <footer class="text-center">
        ${when(
--- a/frontend/src/components/index.ts
+++ b/frontend/src/components/index.ts
@ -46,9 +46,13 @@ import("./queue-exclusion-form").then(({ QueueExclusionForm }) => {
 import("./queue-exclusion-table").then(({ QueueExclusionTable }) => {
  customElements.define("btrix-queue-exclusion-table", QueueExclusionTable);
 });
-import("./numbered-list").then(({ NumberedList }) => {
-  customElements.define("btrix-numbered-list", NumberedList);
-});
+import("./numbered-list").then(
+  ({ NumberedList, NumberedListItem, NumberedListHeader }) => {
+    customElements.define("btrix-numbered-list", NumberedList);
+    customElements.define("btrix-numbered-list-item", NumberedListItem);
+    customElements.define("btrix-numbered-list-header", NumberedListHeader);
+  }
+);
 import("./pagination").then(({ Pagination }) => {
  customElements.define("btrix-pagination", Pagination);
 });
@ -90,6 +94,9 @@ import("./workflow-list").then(({ WorkflowListItem, WorkflowList }) => {
  customElements.define("btrix-workflow-list-item", WorkflowListItem);
  customElements.define("btrix-workflow-list", WorkflowList);
 });
+import("./crawl-logs").then(({ CrawlLogs }) => {
+  customElements.define("btrix-crawl-logs", CrawlLogs);
+});
 import("./section-heading").then(({ SectionHeading }) => {
  customElements.define("btrix-section-heading", SectionHeading);
 });
--- a/frontend/src/components/numbered-list.ts
+++ b/frontend/src/components/numbered-list.ts
@ -1,56 +1,36 @@
-import { LitElement, html, css } from "lit";
-import { property } from "lit/decorators.js";
-import { ifDefined } from "lit/directives/if-defined.js";
-
-type ListItem = {
-  order?: number;
-  style?: string; // inline styles
-  content: any; // any lit template content
-};
-
 /**
 * Styled numbered list
 *
 * Usage example:
 * ```ts
- * <btrix-numbered-list></btrix-numbered-list>
- * ```
- *
- * CSS variables:
- * ```
- * --marker-color
- * --link-color
- * --link-hover-color
+ * <btrix-numbered-list>
+ *   <btrix-numbered-list-item>
+ *     <span slot="marker">1.</span> Content
+ *   </btrix-numbered-list-item>
+ * </btrix-numbered-list>
 * ```
 */
-export class NumberedList extends LitElement {
-  @property({ type: Array })
-  items: ListItem[] = [];
+import { LitElement, html, css } from "lit";
+import { property, queryAssignedElements } from "lit/decorators.js";
+import { classMap } from "lit/directives/class-map.js";
+
+export class NumberedListItem extends LitElement {
+  @property({ type: Boolean })
+  isFirst: boolean = false;
+
+  @property({ type: Boolean })
+  isLast: boolean = false;
+
+  @property({ type: Boolean })
+  isEven: boolean = false;

  static styles = css`
-    :host {
-      display: block;
-    }
-
-    .list {
-      display: grid;
-      grid-template-columns: minmax(6ch, max-content) 1fr;
-      align-items: center;
-      font-family: var(--sl-font-mono);
-      list-style-type: none;
-      margin: 0;
-      padding: 0;
-    }
-
-    .list li {
+    :host,
+    .item {
      display: contents;
    }

-    .item-content {
-      --item-height: 1.5rem;
-      contain: paint;
-      contain-intrinsic-height: auto var(--item-height);
-      content-visibility: auto;
+    .content {
      border-left: var(--sl-panel-border-width) solid
        var(--sl-panel-border-color);
      border-right: var(--sl-panel-border-width) solid
@ -61,57 +41,118 @@ export class NumberedList extends LitElement {
      box-sizing: border-box;
    }

-    li:first-child .item-content {
+    .marker {
+      color: var(--sl-color-neutral-400);
+      line-height: 1;
+      font-size: var(--sl-font-size-medium);
+      font-weight: var(--sl-font-weight-normal);
+      text-align: right;
+      white-space: nowrap;
+    }
+
+    .item.first .content {
      border-top: var(--sl-panel-border-width) solid
        var(--sl-panel-border-color);
      border-top-left-radius: var(--sl-border-radius-medium);
      border-top-right-radius: var(--sl-border-radius-medium);
    }

-    li:last-child .item-content {
+    .item.last .content {
      border-bottom: var(--sl-panel-border-width) solid
        var(--sl-panel-border-color);
      border-bottom-left-radius: var(--sl-border-radius-medium);
      border-bottom-right-radius: var(--sl-border-radius-medium);
    }

-    li:nth-child(even) .item-content {
-      background-color: var(--sl-color-neutral-50);
-    }
-
-    .item-marker {
-      color: var(--marker-color, var(--sl-color-neutral-400));
-      line-height: 1;
-      font-size: var(--sl-font-size-medium);
-      font-weight: var(--sl-font-weight-normal);
-      text-align: right;
-      margin-right: var(--sl-spacing-x-small);
-      white-space: nowrap;
-    }
-
-    a {
-      color: var(--link-color, var(--sl-color-indigo-500));
-      text-decoration: none;
-    }
-
-    a:hover {
-      color: var(--link-hover-color, var(--sl-color-indigo-400));
+    .item.even .content {
+      background-color: var(--sl-color-neutral-50); */
    }
  `;

  render() {
    return html`
-      <ol class="list">
-        ${this.items.map(
-          (item, idx) =>
-            html`
-              <li style=${ifDefined(item.style)}>
-                <div class="item-marker">${item.order || idx + 1}.</div>
-                <div class="item-content">${item.content}</div>
-              </li>
-            `
-        )}
-      </ol>
+      <div
+        class=${classMap({
+          item: true,
+          first: this.isFirst,
+          last: this.isLast,
+          even: this.isEven,
+        })}
+      >
+        <div class="marker"><slot name="marker"></slot></div>
+        <div class="content"><slot></slot></div>
+      </div>
    `;
  }
 }
+
+export class NumberedListHeader extends LitElement {
+  static styles = css`
+    :host,
+    header {
+      display: contents;
+    }
+
+    .content {
+      grid-column: 2 / -1;
+      padding-bottom: var(--sl-spacing-x-small);
+      color: var(--sl-color-neutral-600);
+      font-size: var(--sl-font-size-x-small);
+      line-height: 1rem;
+    }
+  `;
+
+  render() {
+    return html`<header>
+      <div class="content"><slot></slot></div>
+    </header>`;
+  }
+}
+
+export class NumberedList extends LitElement {
+  static styles = css`
+    :host {
+      display: block;
+    }
+
+    .list {
+      display: grid;
+      grid-template-columns: max-content 1fr;
+      grid-column-gap: var(--sl-spacing-x-small);
+      align-items: center;
+    }
+
+    ol {
+      display: contents;
+      font-family: var(--sl-font-mono);
+      list-style-type: none;
+      margin: 0;
+      padding: 0;
+    }
+  `;
+
+  @queryAssignedElements({ selector: "btrix-numbered-list-item" })
+  listItems!: NumberedListItem[];
+
+  render() {
+    return html`
+      <div class="list">
+        <slot name="header"></slot>
+        <ol>
+          <slot @slotchange=${this.handleSlotchange}></slot>
+        </ol>
+      </div>
+    `;
+  }
+
+  private handleSlotchange() {
+    this.listItems.forEach((el, i) => {
+      if (!el.attributes.getNamedItem("role")) {
+        el.setAttribute("role", "listitem");
+      }
+      el.isFirst = i === 0;
+      el.isLast = i === this.listItems.length - 1;
+      el.isEven = i % 2 !== 0;
+    });
+  }
+}
--- a/frontend/src/components/pagination.ts
+++ b/frontend/src/components/pagination.ts
@ -314,6 +314,7 @@ export class Pagination extends LitElement {
    this.dispatchEvent(
      <PageChangeEvent>new CustomEvent("page-change", {
        detail: { page: page, pages: this.pages },
+        composed: true,
      })
    );
  }
--- a/frontend/src/pages/org/crawl-detail.ts
+++ b/frontend/src/pages/org/crawl-detail.ts
@ -2,14 +2,17 @@ import type { TemplateResult } from "lit";
 import { state, property } from "lit/decorators.js";
 import { when } from "lit/directives/when.js";
 import { ifDefined } from "lit/directives/if-defined.js";
+import { classMap } from "lit/directives/class-map.js";
 import { msg, localized, str } from "@lit/localize";

+import type { PageChangeEvent } from "../../components/pagination";
 import { RelativeDuration } from "../../components/relative-duration";
 import type { AuthState } from "../../utils/AuthService";
 import LiteElement, { html } from "../../utils/LiteElement";
 import { isActive } from "../../utils/crawler";
 import { CopyButton } from "../../components/copy-button";
 import type { Crawl, Workflow } from "./types";
+import { APIPaginatedList } from "../../types/api";

 const SECTIONS = [
  "overview",
@ -22,6 +25,9 @@ const SECTIONS = [
 ] as const;
 type SectionName = (typeof SECTIONS)[number];

+const LOG_LEVEL_VARIANTS = {
+  error: "danger",
+} as const;
 const POLL_INTERVAL_SECONDS = 10;

 /**
@ -55,6 +61,9 @@ export class CrawlDetail extends LiteElement {
  @state()
  private crawl?: Crawl;

+  @state()
+  private logs?: APIPaginatedList;
+
  @state()
  private sectionName: SectionName = "overview";

@ -98,6 +107,7 @@ export class CrawlDetail extends LiteElement {
    }

    this.fetchCrawl();
+    this.fetchCrawlLogs();
  }

  willUpdate(changedProperties: Map<string, any>) {
@ -106,6 +116,7 @@ export class CrawlDetail extends LiteElement {
    if (prevId && prevId !== this.crawlId) {
      // Handle update on URL change, e.g. from re-run
      this.fetchCrawl();
+      this.fetchCrawlLogs();
    } else {
      const prevCrawl = changedProperties.get("crawl");

@ -136,7 +147,12 @@ export class CrawlDetail extends LiteElement {
      case "replay":
        sectionContent = this.renderPanel(
          msg("Replay Crawl"),
-          this.renderReplay()
+          this.renderReplay(),
+          {
+            "overflow-hidden": true,
+            "rounded-lg": true,
+            border: true,
+          }
        );
        break;
      case "files":
@ -146,16 +162,24 @@ export class CrawlDetail extends LiteElement {
        );
        break;
      case "logs":
-        sectionContent = this.renderPanel(msg("Logs"), this.renderLogs());
+        sectionContent = this.renderPanel(msg("Error Logs"), this.renderLogs());
        break;
      case "config":
-        sectionContent = this.renderPanel(msg("Config"), this.renderConfig());
+        sectionContent = this.renderPanel(msg("Config"), this.renderConfig(), {
+          "p-4": true,
+          "rounded-lg": true,
+          border: true,
+        });
        break;
      default:
        sectionContent = html`
          <div class="grid gap-5 grid-cols-1 lg:grid-cols-2">
            <div class="col-span-1 flex flex-col">
-              ${this.renderPanel(msg("Overview"), this.renderOverview())}
+              ${this.renderPanel(msg("Overview"), this.renderOverview(), {
+                "p-4": true,
+                "rounded-lg": true,
+                border: true,
+              })}
            </div>
            <div class="col-span-1 flex flex-col">
              ${this.renderPanel(
@ -183,7 +207,12 @@ export class CrawlDetail extends LiteElement {
                    `
                  )}
                `,
-                this.renderMetadata()
+                this.renderMetadata(),
+                {
+                  "p-4": true,
+                  "rounded-lg": true,
+                  border: true,
+                }
              )}
            </div>
          </div>
@ -319,23 +348,24 @@ export class CrawlDetail extends LiteElement {
            icon: "link-replay",
            label: msg("Replay Crawl"),
          })}
-          ${!this.isActive
-            ? renderNavItem({
-                section: "files",
-                iconLibrary: "default",
-                icon: "folder-fill",
-                label: msg("Files"),
-              })
-            : ""}
+          ${renderNavItem({
+            section: "files",
+            iconLibrary: "default",
+            icon: "folder-fill",
+            label: msg("Files"),
+          })}
+          ${renderNavItem({
+            section: "logs",
+            iconLibrary: "default",
+            icon: "terminal-fill",
+            label: msg("Error Logs"),
+          })}
          ${renderNavItem({
            section: "config",
            iconLibrary: "default",
            icon: "file-code-fill",
            label: msg("Config"),
          })}
-          ${
-            /* renderNavItem({ section: "logs", iconLibrary:"default", icon: "terminal-fill", label: msg("Logs") }) */ ""
-          }
        </ul>
      </nav>
    `;
@ -448,19 +478,7 @@ export class CrawlDetail extends LiteElement {
    `;
  }

-  private renderPanel(title: any, content: any) {
-    let panelContainer;
-    switch (this.sectionName) {
-      case "replay":
-        panelContainer = html`
-          <div class="flex-1 rounded-lg border overflow-hidden">${content}</div>
-        `;
-        break;
-      default:
-        panelContainer = html`
-          <div class="flex-1 rounded-lg border p-5">${content}</div>
-        `;
-    }
+  private renderPanel(title: any, content: any, classes: any = {}) {
    return html`
      <h2
        id="exclusions"
@ -468,7 +486,14 @@ export class CrawlDetail extends LiteElement {
      >
        ${title}
      </h2>
-      ${panelContainer}
+      <div
+        class=${classMap({
+          "flex-1": true,
+          ...classes,
+        })}
+      >
+        ${content}
+      </div>
    `;
  }

@ -566,7 +591,7 @@ export class CrawlDetail extends LiteElement {
              ></replay-web-page>
            </div>`
          : html`
-              <p class="text-sm text-neutral-400">
+              <p class="text-sm text-neutral-400 p-4">
                ${this.isActive
                  ? msg("No files yet.")
                  : msg("No files to replay.")}
@ -755,7 +780,7 @@ ${this.crawl?.notes}
    return html`
      ${this.hasFiles
        ? html`
-            <ul class="border rounded text-sm">
+            <ul class="border rounded-lg text-sm">
              ${this.crawl!.resources!.map(
                (file) => html`
                  <li
@ -795,7 +820,32 @@ ${this.crawl?.notes}
  }

  private renderLogs() {
-    return html`TODO`;
+    if (!this.logs) {
+      return html`<div
+        class="w-full flex items-center justify-center my-24 text-3xl"
+      >
+        <sl-spinner></sl-spinner>
+      </div>`;
+    }
+
+    if (!this.logs.total) {
+      return html`<div class="border rounded-lg p-4">
+        <p class="text-sm text-neutral-400">${msg("No error logs to display.")}</p>
+      </div>`;
+    }
+
+    return html`
+      <btrix-crawl-logs
+        .logs=${this.logs}
+        @page-change=${async (e: PageChangeEvent) => {
+          await this.fetchCrawlLogs({
+            page: e.detail.page,
+          });
+          // Scroll to top of list
+          this.scrollIntoView();
+        }}
+      ></btrix-crawl-logs>
+    `;
  }

  private renderConfig() {
@ -834,6 +884,36 @@ ${this.crawl?.notes}
    return data;
  }

+  private async fetchCrawlLogs(
+    params: Partial<APIPaginatedList> = {}
+  ): Promise<void> {
+    try {
+      this.logs = await this.getCrawlLogs(params);
+    } catch {
+      this.notify({
+        message: msg("Sorry, couldn't retrieve crawl logs at this time."),
+        variant: "danger",
+        icon: "exclamation-octagon",
+      });
+    }
+  }
+
+  private async getCrawlLogs(
+    params: Partial<APIPaginatedList>
+  ): Promise<APIPaginatedList> {
+    const page = params.page || this.logs?.page || 1;
+    const pageSize = params.pageSize || this.logs?.pageSize || 50;
+
+    const data: APIPaginatedList = await this.apiFetch(
+      `${this.crawlsAPIBaseUrl || this.crawlsBaseUrl}/${
+        this.crawlId
+      }/errors?page=${page}&pageSize=${pageSize}`,
+      this.authState!
+    );
+
+    return data;
+  }
+
  private async cancel() {
    if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) {
      const data = await this.apiFetch(
@ -973,6 +1053,8 @@ ${this.crawl?.notes}
  private crawlDone() {
    if (!this.crawl) return;

+    this.fetchCrawlLogs();
+
    this.notify({
      message: msg(html`Done crawling <strong>${this.renderName()}</strong>.`),
      variant: "success",
--- a/frontend/src/utils/css.ts
+++ b/frontend/src/utils/css.ts
@ -18,10 +18,9 @@ export const srOnly = css`
  }
 `;

-// From https://tailwindcss.com/docs/text-overflow#truncate
 export const truncate = css`
  .truncate {
-    overflow: hidden;
+    overflow: clip visible;
    text-overflow: ellipsis;
    white-space: nowrap;
  }