From 9ed216ba05afacd367c2e5b256e4aa2a5b8b34e3 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Sat, 22 Jan 2022 14:18:02 -0800 Subject: [PATCH] Run and delete crawl templates from list view (#94) --- .../src/pages/archive/crawl-templates-list.ts | 297 +++++++++++++++++- .../src/pages/archive/crawl-templates-new.ts | 13 +- frontend/src/pages/archive/index.ts | 26 +- frontend/src/pages/archive/types.ts | 15 +- 4 files changed, 302 insertions(+), 49 deletions(-) diff --git a/frontend/src/pages/archive/crawl-templates-list.ts b/frontend/src/pages/archive/crawl-templates-list.ts index 1ea44b02..f3074cc2 100644 --- a/frontend/src/pages/archive/crawl-templates-list.ts +++ b/frontend/src/pages/archive/crawl-templates-list.ts @@ -1,9 +1,26 @@ import { state, property } from "lit/decorators.js"; import { msg, localized, str } from "@lit/localize"; +import cronParser from "cron-parser"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; -import type { CrawlTemplate } from "./types"; +import type { CrawlConfig } from "./types"; + +type CrawlTemplate = { + id: string; + name: string; + schedule: string; + user: string; + crawlCount: number; + lastCrawlId: string; + lastCrawlTime: string; + currCrawlId: string; + config: CrawlConfig; +}; +type RunningCrawlsMap = { + /** Map of configId: crawlId */ + [configId: string]: string; +}; /** * Usage: @@ -19,31 +36,289 @@ export class CrawlTemplatesList extends LiteElement { @property({ type: String }) archiveId!: string; - @property({ type: Array }) + @state() crawlTemplates?: CrawlTemplate[]; @state() - private serverError?: string; + runningCrawlsMap: RunningCrawlsMap = {}; + + private get timeZone() { + return Intl.DateTimeFormat().resolvedOptions().timeZone; + } + + async firstUpdated() { + try { + this.crawlTemplates = await this.getCrawlTemplates(); + if (!this.crawlTemplates.length) { + this.navTo(`/archives/${this.archiveId}/crawl-templates/new`); + } + } catch (e) { + this.notify({ + message: msg("Sorry, couldn't retrieve crawl templates at this time."), + type: "danger", + icon: "exclamation-octagon", + duration: 10000, + }); + } + } render() { + if (!this.crawlTemplates) { + return html`
+ +
`; + } + return html` -
-
+ +
+
this.navTo(`/archives/${this.archiveId}/crawl-templates/new`)} + role="button" > - - ${msg("Create new crawl template")} - + + ${msg("Create New Crawl Template")} +
-
- ${this.crawlTemplates?.map( - (template) => html`
${template.id}
` +
+ ${this.crawlTemplates.map( + (t) => + html`
+
+ + ${t.name || "?"} + + + + + +
    + +
+ +
+ +
+
+
+ ${t.config.seeds.join(", ")} +
+
+ ${t.crawlCount === 1 + ? msg(str`${t.crawlCount} crawl`) + : msg( + str`${(t.crawlCount || 0).toLocaleString()} crawls` + )} +
+
+ ${msg(html`Last: + `)} +
+
+ ${t.schedule + ? msg(html`Next: + `) + : html`${msg("No schedule")}`} +
+
+
+ +
+
+
` )}
`; } + + /** + * Fetch crawl templates and record running crawls + * associated with the crawl templates + **/ + private async getCrawlTemplates(): Promise { + type CrawlConfig = Omit & { + config: Omit & { + seeds: (string | { url: string })[]; + }; + }; + + const data: { crawlConfigs: CrawlConfig[] } = await this.apiFetch( + `/archives/${this.archiveId}/crawlconfigs`, + this.authState! + ); + + const crawlConfigs: CrawlTemplate[] = []; + const runningCrawlsMap: RunningCrawlsMap = {}; + + data.crawlConfigs.forEach(({ config, ...configMeta }) => { + crawlConfigs.push({ + ...configMeta, + config: { + ...config, + // Flatten seeds into array of URL strings + seeds: config.seeds.map((seed) => + typeof seed === "string" ? seed : seed.url + ), + }, + }); + + if (configMeta.currCrawlId) { + runningCrawlsMap[configMeta.id] = configMeta.currCrawlId; + } + }); + + this.runningCrawlsMap = runningCrawlsMap; + + return crawlConfigs; + } + + private async deleteTemplate(template: CrawlTemplate): Promise { + try { + await this.apiFetch( + `/archives/${this.archiveId}/crawlconfigs/${template.id}`, + this.authState!, + { + method: "DELETE", + } + ); + + this.notify({ + message: msg(str`Deleted ${template.name}.`), + type: "success", + icon: "check2-circle", + }); + + this.crawlTemplates = this.crawlTemplates!.filter( + (t) => t.id !== template.id + ); + } catch { + this.notify({ + message: msg("Sorry, couldn't delete crawl template at this time."), + type: "danger", + icon: "exclamation-octagon", + }); + } + } + + private async runNow(template: CrawlTemplate): Promise { + try { + const data = await this.apiFetch( + `/archives/${this.archiveId}/crawlconfigs/${template.id}/run`, + this.authState!, + { + method: "POST", + } + ); + + const crawlId = data.started; + + this.runningCrawlsMap = { + ...this.runningCrawlsMap, + [template.id]: crawlId, + }; + + this.notify({ + message: msg( + str`Started crawl from ${template.name}.
View crawl` + ), + type: "success", + icon: "check2-circle", + duration: 10000, + }); + } catch { + this.notify({ + message: msg("Sorry, couldn't run crawl at this time."), + type: "danger", + icon: "exclamation-octagon", + }); + } + } } customElements.define("btrix-crawl-templates-list", CrawlTemplatesList); diff --git a/frontend/src/pages/archive/crawl-templates-new.ts b/frontend/src/pages/archive/crawl-templates-new.ts index 0959f189..c33ac5f2 100644 --- a/frontend/src/pages/archive/crawl-templates-new.ts +++ b/frontend/src/pages/archive/crawl-templates-new.ts @@ -5,7 +5,16 @@ import cronParser from "cron-parser"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import { getLocaleTimeZone } from "../../utils/localization"; -import type { CrawlTemplate } from "./types"; +import type { CrawlConfig } from "./types"; + +export type NewCrawlTemplate = { + id?: string; + name: string; + schedule: string; + runNow: boolean; + crawlTimeout?: number; + config: CrawlConfig; +}; const initialValues = { name: "", @@ -448,7 +457,7 @@ export class CrawlTemplatesNew extends LiteElement { const crawlTimeoutMinutes = formData.get("crawlTimeoutMinutes"); const pageLimit = formData.get("limit"); const seedUrlsStr = formData.get("seedUrls"); - const template: Partial = { + const template: Partial = { name: formData.get("name") as string, schedule: this.getUTCSchedule(), runNow: this.isRunNow, diff --git a/frontend/src/pages/archive/index.ts b/frontend/src/pages/archive/index.ts index e9848d33..d1be0083 100644 --- a/frontend/src/pages/archive/index.ts +++ b/frontend/src/pages/archive/index.ts @@ -7,7 +7,6 @@ import type { ArchiveData } from "../../utils/archives"; import LiteElement, { html } from "../../utils/LiteElement"; import { needLogin } from "../../utils/auth"; import { isOwner } from "../../utils/archives"; -import type { CrawlTemplate } from "./types"; import "./crawl-templates-list"; import "./crawl-templates-new"; @@ -40,9 +39,6 @@ export class Archive extends LiteElement { @state() private archive?: ArchiveData; - @state() - private crawlTemplates?: CrawlTemplate[]; - @state() private successfullyInvitedEmail?: string; @@ -61,17 +57,7 @@ export class Archive extends LiteElement { } async updated(changedProperties: any) { - if ( - changedProperties.has("archiveTab") && - this.archiveTab === "crawl-templates" && - !this.isNewResourceTab - ) { - this.crawlTemplates = await this.getCrawlTemplates(); - - if (!this.crawlTemplates.length) { - this.navTo(`/archives/${this.archiveId}/crawl-templates/new`); - } - } else if (changedProperties.has("isAddingMember") && this.isAddingMember) { + if (changedProperties.has("isAddingMember") && this.isAddingMember) { this.successfullyInvitedEmail = undefined; } } @@ -178,7 +164,6 @@ export class Archive extends LiteElement { return html``; } @@ -267,15 +252,6 @@ export class Archive extends LiteElement { return data; } - async getCrawlTemplates(): Promise { - const data = await this.apiFetch( - `/archives/${this.archiveId}/crawlconfigs`, - this.authState! - ); - - return data.crawl_configs; - } - onInviteSuccess( event: CustomEvent<{ inviteEmail: string; isExistingUser: boolean }> ) { diff --git a/frontend/src/pages/archive/types.ts b/frontend/src/pages/archive/types.ts index a230f6ca..d56a9966 100644 --- a/frontend/src/pages/archive/types.ts +++ b/frontend/src/pages/archive/types.ts @@ -1,12 +1,5 @@ -export type CrawlTemplate = { - id?: string; - name: string; - schedule: string; - runNow: boolean; - crawlTimeout?: number; - config: { - seeds: string[]; - scopeType?: string; - limit?: number; - }; +export type CrawlConfig = { + seeds: string[]; + scopeType?: string; + limit?: number; };