From be4bf3742f0df2f55d02057ed9244c2e77324f10 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Sun, 30 Jan 2022 18:36:43 -0800 Subject: [PATCH] Initial crawl detail page (#108) --- frontend/src/components/index.ts | 3 + frontend/src/components/relative-duration.ts | 32 ++ frontend/src/index.ts | 2 + frontend/src/pages/archive/crawl-detail.ts | 447 ++++++++++++++++++ .../pages/archive/crawl-templates-detail.ts | 8 +- .../src/pages/archive/crawl-templates-list.ts | 4 +- .../src/pages/archive/crawl-templates-new.ts | 2 +- frontend/src/pages/archive/crawl-templates.ts | 2 +- frontend/src/pages/archive/crawls-list.ts | 119 ++--- frontend/src/pages/archive/index.ts | 12 + frontend/src/pages/archive/types.ts | 19 + frontend/src/routes.ts | 1 + 12 files changed, 572 insertions(+), 79 deletions(-) create mode 100644 frontend/src/components/relative-duration.ts create mode 100644 frontend/src/pages/archive/crawl-detail.ts diff --git a/frontend/src/components/index.ts b/frontend/src/components/index.ts index fe23527f..39270c58 100644 --- a/frontend/src/components/index.ts +++ b/frontend/src/components/index.ts @@ -15,6 +15,9 @@ import("./copy-button").then(({ CopyButton }) => { import("./invite-form").then(({ InviteForm }) => { customElements.define("btrix-invite-form", InviteForm); }); +import("./relative-duration").then(({ RelativeDuration }) => { + customElements.define("btrix-relative-duration", RelativeDuration); +}); import("./sign-up-form").then(({ SignUpForm }) => { customElements.define("btrix-sign-up-form", SignUpForm); }); diff --git a/frontend/src/components/relative-duration.ts b/frontend/src/components/relative-duration.ts new file mode 100644 index 00000000..2ade2a5f --- /dev/null +++ b/frontend/src/components/relative-duration.ts @@ -0,0 +1,32 @@ +import { LitElement } from "lit"; +import { property, state } from "lit/decorators.js"; +import humanizeDuration from "pretty-ms"; + +/** + * Show time passed from date in human-friendly format + * + * Usage example: + * ```ts + * + * ``` + * + * @event on-copied + */ +export class RelativeDuration extends LitElement { + @property({ type: String }) + value?: string; // `new Date` compatible date format + + static humanize(duration: number) { + return humanizeDuration(duration, { + secondsDecimalDigits: 0, + }); + } + + render() { + if (!this.value) return ""; + + return RelativeDuration.humanize( + Date.now() - new Date(this.value).valueOf() + ); + } +} diff --git a/frontend/src/index.ts b/frontend/src/index.ts index cdac8759..bf8b3d4f 100644 --- a/frontend/src/index.ts +++ b/frontend/src/index.ts @@ -371,6 +371,7 @@ export class App extends LiteElement { case "archive": case "archiveAddMember": case "archiveNewResourceTab": + case "crawl": case "crawlTemplate": case "crawlTemplateEdit": return appLayout(html` + * ``` + */ +@localized() +export class CrawlDetail extends LiteElement { + @property({ type: Object }) + authState?: AuthState; + + @property({ type: String }) + archiveId?: string; + + @property({ type: String }) + crawlId?: string; + + @state() + private crawl?: Crawl; + + @state() + private watchUrl?: string; + + @state() + private isWatchExpanded: boolean = false; + + // For long polling: + private timerId?: number; + + // TODO localize + private numberFormatter = new Intl.NumberFormat(); + + async firstUpdated() { + this.fetchCrawl(); + + // try { + // this.watchUrl = await this.watchCrawl(); + // console.log(this.watchUrl); + // } catch (e) { + // console.error(e); + // } + } + + disconnectedCallback(): void { + this.stopPollTimer(); + super.disconnectedCallback(); + } + + render() { + return html` +
+

+ ${this.crawl?.id || + html``} +

+
+ +
+
+
+ ${this.renderWatch()} +
+ +
+ ${this.renderDetails()} +
+
+ +
+

${msg("Files")}

+ ${this.renderFiles()} +
+
+ `; + } + + private renderWatch() { + const isRunning = this.crawl?.state === "running"; + + return html` +
+ + [watch/replay] +
+
+ ${this.isWatchExpanded + ? html` + (this.isWatchExpanded = false)} + > + ` + : html` + (this.isWatchExpanded = true)} + > + `} + ${this.watchUrl + ? html` + + ` + : ""} +
+ `; + } + + private renderDetails() { + const isRunning = this.crawl?.state === "running"; + + return html` +
+
+
${msg("Crawl Template")}
+
+ ${this.crawl + ? html` + ${this.crawl.configName} + ` + : html``} +
+
+ +
+
${msg("Status")}
+
+ ${this.crawl + ? html` +
+
+ + ● + + ${this.crawl.state.replace(/_/g, " ")} +
+
+ ` + : html``} + ${isRunning + ? html` + + + ${msg("Manage")} + + +
+ + ${msg("Stop Crawl")} + + + ${msg("Cancel Crawl")} + +
+ + ` + : ""} +
+
+
+
${msg("Pages Crawled")}
+
+ ${this.crawl?.stats + ? html` + + ${this.numberFormatter.format(+this.crawl.stats.done)} + / + ${this.numberFormatter.format(+this.crawl.stats.found)} + + ` + : html``} +
+
+
+
${msg("Run Duration")}
+
+ ${this.crawl + ? html` + ${this.crawl.finished + ? html`${RelativeDuration.humanize( + new Date(`${this.crawl.finished}Z`).valueOf() - + new Date(`${this.crawl.started}Z`).valueOf() + )}` + : html` + + + + `} + ` + : html``} +
+
+
+
${msg("Started")}
+
+ ${this.crawl + ? html` + + ` + : html``} +
+
+
+
${msg("Finished")}
+
+ ${this.crawl + ? html` + ${this.crawl.finished + ? html`` + : html`${msg("Pending")}`} + ` + : html``} +
+
+
+
${msg("Reason")}
+
+ ${this.crawl + ? html` + ${this.crawl.manual + ? msg( + html`Manual start by + ${this.crawl?.userName || this.crawl?.userid}` + ) + : msg(html`Scheduled run`)} + ` + : html``} +
+
+
+ `; + } + + private renderFiles() { + return html` + + `; + } + + /** + * Fetch crawl and update internal state + */ + private async fetchCrawl(): Promise { + try { + this.crawl = await this.getCrawl(); + + if (this.crawl.state === "running") { + // Start timer for next poll + this.timerId = window.setTimeout(() => { + this.fetchCrawl(); + }, 1000 * POLL_INTERVAL_SECONDS); + } else { + this.stopPollTimer(); + } + } catch { + this.notify({ + message: msg("Sorry, couldn't retrieve crawl at this time."), + type: "danger", + icon: "exclamation-octagon", + }); + } + } + + async getCrawl(): Promise { + // Mock to use in dev: + // return import("../../__mocks__/api/archives/[id]/crawls").then( + // (module) => module.default.running[0] + // // (module) => module.default.finished[0] + // ); + + const data: Crawl = await this.apiFetch( + `/archives/${this.archiveId}/crawls/${this.crawlId}`, + this.authState! + ); + + return data; + } + + private async watchCrawl(): Promise { + const data = await this.apiFetch( + `/archives/${this.archiveId}/crawls/${this.crawlId}/watch`, + this.authState!, + { + method: "POST", + } + ); + + return data.watch_url; + } + + private async cancel() { + if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) { + const data = await this.apiFetch( + `/archives/${this.archiveId}/crawls/${this.crawlId}/cancel`, + this.authState!, + { + method: "POST", + } + ); + + if (data.canceled === true) { + this.fetchCrawl(); + } else { + this.notify({ + message: msg("Sorry, couldn't cancel crawl at this time."), + type: "danger", + icon: "exclamation-octagon", + }); + } + } + } + + private async stop() { + if (window.confirm(msg("Are you sure you want to stop the crawl?"))) { + const data = await this.apiFetch( + `/archives/${this.archiveId}/crawls/${this.crawlId}/stop`, + this.authState!, + { + method: "POST", + } + ); + + if (data.stopped_gracefully === true) { + this.fetchCrawl(); + } else { + this.notify({ + message: msg("Sorry, couldn't stop crawl at this time."), + type: "danger", + icon: "exclamation-octagon", + }); + } + } + } + + private stopPollTimer() { + window.clearTimeout(this.timerId); + } +} + +customElements.define("btrix-crawl-detail", CrawlDetail); diff --git a/frontend/src/pages/archive/crawl-templates-detail.ts b/frontend/src/pages/archive/crawl-templates-detail.ts index 69f65863..00dcb561 100644 --- a/frontend/src/pages/archive/crawl-templates-detail.ts +++ b/frontend/src/pages/archive/crawl-templates-detail.ts @@ -272,7 +272,7 @@ export class CrawlTemplatesDetail extends LiteElement { ${this.crawlTemplate.currCrawlId ? html` ${msg("View crawl")}` @@ -298,7 +298,7 @@ export class CrawlTemplatesDetail extends LiteElement { ${this.crawlTemplate?.lastCrawlId ? html`${msg("View crawl")} @@ -330,7 +330,7 @@ export class CrawlTemplatesDetail extends LiteElement { return html` ${msg("View currently running crawl")} @@ -417,7 +417,7 @@ export class CrawlTemplatesDetail extends LiteElement { this.crawlTemplate!.name }.
View crawl` + }/crawls/crawl/${data.run_now_job}">View crawl` ), type: "success", icon: "check2-circle", diff --git a/frontend/src/pages/archive/crawl-templates-list.ts b/frontend/src/pages/archive/crawl-templates-list.ts index ec57ce07..d28aa653 100644 --- a/frontend/src/pages/archive/crawl-templates-list.ts +++ b/frontend/src/pages/archive/crawl-templates-list.ts @@ -280,7 +280,7 @@ export class CrawlTemplatesList extends LiteElement { e.stopPropagation(); this.runningCrawlsMap[t.id] ? this.navTo( - `/archives/${this.archiveId}/crawls/${ + `/archives/${this.archiveId}/crawls/crawl/${ this.runningCrawlsMap[t.id] }` ) @@ -415,7 +415,7 @@ export class CrawlTemplatesList extends LiteElement { this.notify({ message: msg( - str`Started crawl from ${template.name}.
View crawl` + str`Started crawl from ${template.name}.
View crawl` ), type: "success", icon: "check2-circle", diff --git a/frontend/src/pages/archive/crawl-templates-new.ts b/frontend/src/pages/archive/crawl-templates-new.ts index e131ef32..dbe34a38 100644 --- a/frontend/src/pages/archive/crawl-templates-new.ts +++ b/frontend/src/pages/archive/crawl-templates-new.ts @@ -543,7 +543,7 @@ export class CrawlTemplatesNew extends LiteElement { this.notify({ message: data.run_now_job ? msg( - str`Crawl running with new template.
View crawl` + str`Crawl running with new template.
View crawl` ) : msg("Crawl template created."), type: "success", diff --git a/frontend/src/pages/archive/crawl-templates.ts b/frontend/src/pages/archive/crawl-templates.ts index a045a684..e2db9b3a 100644 --- a/frontend/src/pages/archive/crawl-templates.ts +++ b/frontend/src/pages/archive/crawl-templates.ts @@ -155,7 +155,7 @@ export class CrawlTemplatesList extends LiteElement { this.notify({ message: msg( - str`Started crawl from ${template.name}.
View crawl` + str`Started crawl from ${template.name}.
View crawl` ), type: "success", icon: "check2-circle", diff --git a/frontend/src/pages/archive/crawls-list.ts b/frontend/src/pages/archive/crawls-list.ts index a033f153..e813e065 100644 --- a/frontend/src/pages/archive/crawls-list.ts +++ b/frontend/src/pages/archive/crawls-list.ts @@ -1,6 +1,5 @@ import { state, property } from "lit/decorators.js"; import { msg, localized, str } from "@lit/localize"; -import humanizeDuration from "pretty-ms"; import debounce from "lodash/fp/debounce"; import flow from "lodash/fp/flow"; import map from "lodash/fp/map"; @@ -8,28 +7,10 @@ import orderBy from "lodash/fp/orderBy"; import Fuse from "fuse.js"; import { CopyButton } from "../../components/copy-button"; +import { RelativeDuration } from "../../components/relative-duration"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; - -type Crawl = { - id: string; - user: string; - username?: string; - aid: string; - cid: string; - configName?: string; - schedule: string; - manual: boolean; - started: string; // UTC ISO date - finished?: string; // UTC ISO date - state: string; // "running" | "complete" | "failed" | "partial_complete" - scale: number; - stats: { done: number; found: number } | null; - files?: { filename: string; hash: string; size: number }[]; - fileCount?: number; - fileSize?: number; - completions?: number; -}; +import type { Crawl } from "./types"; type CrawlSearchResult = { item: Crawl; @@ -119,8 +100,8 @@ export class CrawlsList extends LiteElement { } disconnectedCallback(): void { - super.disconnectedCallback(); this.stopPollTimer(); + super.disconnectedCallback(); } render() { @@ -254,17 +235,14 @@ export class CrawlsList extends LiteElement { private renderCrawlItem = ({ item: crawl }: CrawlSearchResult) => { return html`
  • + this.navTo(`/archives/${this.archiveId}/crawls/crawl/${crawl.id}`)} + title=${crawl.configName || crawl.cid} >
    -
    - ${crawl.configName || crawl.cid} -
    +
    ${crawl.configName || crawl.cid}
    - ${crawl.manual - ? html` ${msg("Manual Start")}` - : html` - ${msg("Scheduled Run")} - `}
    -
    +
    ` - : humanizeDuration( - Date.now() - new Date(`${crawl.started}Z`).valueOf(), - { - secondsDecimalDigits: 0, - } - )} + : html``}
    -
    +
    ${crawl.finished ? html`
    @@ -345,12 +309,9 @@ export class CrawlsList extends LiteElement {
    ${msg( - str`in ${humanizeDuration( + str`in ${RelativeDuration.humanize( new Date(`${crawl.finished}Z`).valueOf() - - new Date(`${crawl.started}Z`).valueOf(), - { - secondsDecimalDigits: 0, - } + new Date(`${crawl.started}Z`).valueOf() )}` )}
    @@ -360,9 +321,9 @@ export class CrawlsList extends LiteElement {
    - ${this.numberFormatter.format(crawl.stats.done)} + ${this.numberFormatter.format(+crawl.stats.done)} / - ${this.numberFormatter.format(crawl.stats.found)} + ${this.numberFormatter.format(+crawl.stats.found)}
    ${msg("pages crawled")} @@ -370,20 +331,30 @@ export class CrawlsList extends LiteElement { ` : ""}
    -
    +
    ${crawl.manual ? html` -
    - ${msg("Started by")} +
    + ${msg("Manual Start")}
    -
    - ${crawl.username || crawl.user} +
    + ${msg(str`by ${crawl.userName || crawl.userid}`)}
    ` - : ""} + : html` +
    + ${msg("Scheduled Run")} +
    + `}
    - + e.stopPropagation()}> -