From 7777a228292539a62cd344574ab720b9067efbd6 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Sat, 29 Jan 2022 14:37:16 -0800 Subject: [PATCH] Poll crawls list & add additional details (#116) --- frontend/src/pages/archive/crawls-list.ts | 168 +++++++++++++++++----- frontend/src/shoelace.ts | 3 + 2 files changed, 138 insertions(+), 33 deletions(-) diff --git a/frontend/src/pages/archive/crawls-list.ts b/frontend/src/pages/archive/crawls-list.ts index b27ec9c5..ed2f0f78 100644 --- a/frontend/src/pages/archive/crawls-list.ts +++ b/frontend/src/pages/archive/crawls-list.ts @@ -14,8 +14,10 @@ import LiteElement, { html } from "../../utils/LiteElement"; type Crawl = { id: string; user: string; + username?: string; aid: string; cid: string; + configName?: string; schedule: string; manual: boolean; started: string; // UTC ISO date @@ -24,6 +26,8 @@ type Crawl = { scale: number; stats: { done: number; found: number } | null; files?: { filename: string; hash: string; size: number }[]; + fileCount?: number; + fileSize?: number; completions?: number; }; @@ -31,12 +35,16 @@ type CrawlSearchResult = { item: Crawl; }; +const POLL_INTERVAL_SECONDS = 10; const MIN_SEARCH_LENGTH = 2; const sortableFieldLabels = { started_desc: msg("Newest"), started_asc: msg("Oldest"), state: msg("Status"), + configName: msg("Crawl Template Name"), cid: msg("Crawl Template ID"), + fileSize_asc: msg("Smallest Files"), + fileSize_desc: msg("Largest Files"), }; function isRunning(crawl: Crawl) { @@ -83,7 +91,16 @@ export class CrawlsList extends LiteElement { private filterBy: string = ""; // For fuzzy search: - private fuse = new Fuse([], { keys: ["cid"], shouldSort: false }); + private fuse = new Fuse([], { + keys: ["cid", "configName"], + shouldSort: false, + }); + + // For long polling: + private timerId?: number; + + // TODO localize + private numberFormatter = new Intl.NumberFormat(); private sortCrawls(crawls: CrawlSearchResult[]): CrawlSearchResult[] { return orderBy(({ item }) => item[this.orderBy.field])( @@ -92,11 +109,20 @@ export class CrawlsList extends LiteElement { } protected updated(changedProperties: Map) { - if (this.shouldFetch && changedProperties.has("shouldFetch")) { - this.fetchCrawls(); + if (changedProperties.has("shouldFetch")) { + if (this.shouldFetch) { + this.fetchCrawls(); + } else { + this.stopPollTimer(); + } } } + disconnectedCallback(): void { + super.disconnectedCallback(); + this.stopPollTimer(); + } + render() { if (!this.crawls) { return html`
-
+
${msg("Sort by")}
{ return html`
  • -
    -
    - ${crawl.id} -
    -
    +
    +
    ${crawl.cid}${crawl.configName || crawl.cid}
    +
    + + ${crawl.manual + ? html` ${msg("Manual Start")}` + : html` + ${msg("Scheduled Run")} + `} +
    -
    +
    ` : humanizeDuration( @@ -271,24 +314,60 @@ export class CrawlsList extends LiteElement {
    -
    -
    - ${crawl.manual - ? msg(html`Manual start by ${crawl.user}`) - : msg(html`Scheduled run`)} -
    - -
    - -
    +
    + ${crawl.finished + ? html` +
    + + + + + (${crawl.fileCount === 1 + ? msg(str`${crawl.fileCount} file`) + : msg(str`${crawl.fileCount} files`)}) + +
    +
    + ${msg( + str`in ${humanizeDuration( + new Date(`${crawl.finished}Z`).valueOf() - + new Date(`${crawl.started}Z`).valueOf(), + { + secondsDecimalDigits: 0, + } + )}` + )} +
    + ` + : crawl.stats + ? html` +
    + ${this.numberFormatter.format(crawl.stats.done)} + / + ${this.numberFormatter.format(crawl.stats.found)} +
    +
    + ${msg("pages crawled")} +
    + ` + : ""} +
    +
    + ${crawl.manual + ? html` +
    + ${msg("Started by")} +
    +
    + ${crawl.username || crawl.user} +
    + ` + : ""}
    @@ -337,6 +416,18 @@ export class CrawlsList extends LiteElement { > ${msg("Copy Crawl Template ID")}
    • +
    @@ -351,12 +442,19 @@ export class CrawlsList extends LiteElement { * Fetch crawls and update internal state */ private async fetchCrawls(): Promise { + if (!this.shouldFetch) return; + try { const { crawls } = await this.getCrawls(); this.crawls = crawls; // Update search/filter collection this.fuse.setCollection(this.crawls as any); + + // Start timer for next poll + this.timerId = window.setTimeout(() => { + this.fetchCrawls(); + }, 1000 * POLL_INTERVAL_SECONDS); } catch (e) { this.notify({ message: msg("Sorry, couldn't retrieve crawls at this time."), @@ -366,6 +464,10 @@ export class CrawlsList extends LiteElement { } } + private stopPollTimer() { + window.clearTimeout(this.timerId); + } + private async getCrawls(): Promise<{ crawls: Crawl[] }> { // Mock to use in dev: // return import("../../__mocks__/api/archives/[id]/crawls").then( diff --git a/frontend/src/shoelace.ts b/frontend/src/shoelace.ts index c9b2a2e4..bf22253e 100644 --- a/frontend/src/shoelace.ts +++ b/frontend/src/shoelace.ts @@ -20,6 +20,9 @@ import( import( /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/form/form" ); +import( + /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/format-bytes/format-bytes" +); import( /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/format-date/format-date" );