import type { TemplateResult } from "lit"; import { state, property, customElement } from "lit/decorators.js"; import { when } from "lit/directives/when.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { classMap } from "lit/directives/class-map.js"; import { msg, localized, str } from "@lit/localize"; import type { PageChangeEvent } from "@/components/ui/pagination"; import { RelativeDuration } from "@/components/ui/relative-duration"; import type { AuthState } from "@/utils/AuthService"; import LiteElement, { html } from "@/utils/LiteElement"; import { isActive } from "@/utils/crawler"; import { CopyButton } from "@/components/ui/copy-button"; import type { ArchivedItem, Crawl, CrawlConfig, Seed, Workflow } from "./types"; import type { APIPaginatedList } from "@/types/api"; import { humanizeExecutionSeconds } from "@/utils/executionTimeFormatter"; import type { CrawlLog } from "@/features/archived-items/crawl-logs"; import capitalize from "lodash/fp/capitalize"; const SECTIONS = [ "overview", "watch", "replay", "files", "logs", "config", "exclusions", ] as const; type SectionName = (typeof SECTIONS)[number]; /** * Usage: * ```ts * * ``` */ @localized() @customElement("btrix-crawl-detail") export class CrawlDetail extends LiteElement { @property({ type: Object }) authState?: AuthState; @property({ type: String }) itemType: ArchivedItem["type"] = "crawl"; @property({ type: String }) collectionId?: string; @property({ type: String }) workflowId?: string; @property({ type: Boolean }) showOrgLink = false; @property({ type: String }) orgId!: string; @property({ type: String }) crawlId!: string; @property({ type: Boolean }) isCrawler!: boolean; @state() private crawl?: ArchivedItem; @state() private workflow?: Workflow; @state() private seeds?: APIPaginatedList; @state() private logs?: APIPaginatedList; @state() private sectionName: SectionName = "overview"; @state() private openDialogName?: "scale" | "metadata" | "exclusions"; private get listUrl(): string { let path = "items"; if (this.workflowId) { path = `workflows/crawl/${this.workflowId}#crawls`; } else if (this.collectionId) { path = `collections/view/${this.collectionId}/items`; } else if (this.crawl?.type === "upload") { path = "items/upload"; } else if (this.crawl?.type === "crawl") { path = "items/crawl"; } return `${this.orgBasePath}/${path}`; } // TODO localize private numberFormatter = new Intl.NumberFormat(); private get isActive(): boolean | null { if (!this.crawl) return null; return ( this.crawl.state === "running" || this.crawl.state === "starting" || this.crawl.state === "waiting_capacity" || this.crawl.state === "waiting_org_limit" || this.crawl.state === "stopping" ); } private get hasFiles(): boolean | null { if (!this.crawl) return null; if (!this.crawl.resources) return false; return this.crawl.resources.length > 0; } willUpdate(changedProperties: Map) { if (changedProperties.has("crawlId") && this.crawlId) { this.fetchCrawl(); this.fetchCrawlLogs(); this.fetchSeeds(); } if (changedProperties.has("workflowId") && this.workflowId) { this.fetchWorkflow(); } } connectedCallback(): void { // Set initial active section based on URL #hash value const hash = window.location.hash.slice(1); if ((SECTIONS as readonly string[]).includes(hash)) { this.sectionName = hash as SectionName; } super.connectedCallback(); } render() { const authToken = this.authState!.headers.Authorization.split(" ")[1]; let sectionContent: string | TemplateResult = ""; switch (this.sectionName) { case "replay": sectionContent = this.renderPanel(msg("Replay"), this.renderReplay(), { "overflow-hidden": true, "rounded-lg": true, border: true, }); break; case "files": sectionContent = this.renderPanel(msg("Files"), this.renderFiles()); break; case "logs": sectionContent = this.renderPanel( html` ${this.renderTitle(msg("Error Logs"))} ${msg("Download Logs")} `, this.renderLogs() ); break; case "config": sectionContent = this.renderPanel( msg("Crawl Settings"), this.renderConfig(), { "p-4": true, "rounded-lg": true, border: true, } ); break; default: sectionContent = html`
${this.renderPanel(msg("Overview"), this.renderOverview(), { "p-4": true, "rounded-lg": true, border: true, })}
${this.renderPanel( html` ${this.renderTitle(msg("Metadata"))} ${when( this.isCrawler, () => html` ` )} `, this.renderMetadata(), { "p-4": true, "rounded-lg": true, border: true, } )}
`; break; } let label = "Back"; if (this.workflowId) { label = msg("Back to Crawl Workflow"); } else if (this.collectionId) { label = msg("Back to Collection"); } else if (this.crawl) { if (this.crawl.type === "upload") { label = msg("Back to All Uploads"); } else if (this.crawl.type === "crawl") { label = msg("Back to All Crawls"); } else { label = msg("Back to Archived Items"); } } return html`
${this.renderHeader()}
${this.renderNav()}
${sectionContent}
(this.openDialogName = undefined)} @updated=${() => this.fetchCrawl()} > `; } private renderName() { if (!this.crawl) return html``; if (this.crawl.name) return this.crawl.name; if (!this.crawl.firstSeed || !this.crawl.seedCount) return this.crawl.id; const remainder = this.crawl.seedCount - 1; let crawlName: TemplateResult = html`${this.crawl.firstSeed}`; if (remainder) { if (remainder === 1) { crawlName = msg( html`${this.crawl.firstSeed} +${remainder} URL` ); } else { crawlName = msg( html`${this.crawl.firstSeed} +${remainder} URLs` ); } } return crawlName; } private renderNav() { const renderNavItem = ({ section, label, iconLibrary, icon, }: { section: SectionName; label: any; iconLibrary: "app" | "default"; icon: string; }) => { const isActive = section === this.sectionName; const baseUrl = window.location.pathname.split("#")[0]; return html` `; }; return html` `; } private renderHeader() { return html`

${this.renderName()}

${this.isActive ? html` ${msg("Stop")} ${msg("Cancel")} ` : ""} ${this.crawl && this.isCrawler ? this.renderMenu() : html``}
`; } private renderMenu() { if (!this.crawl) return; return html` ${this.isActive ? html`` : msg("Actions")} ${when( this.isCrawler, () => html` { this.openMetadataEditor(); }} > ${msg("Edit Metadata")} ` )} ${when( this.itemType === "crawl", () => html` this.navTo( `${this.orgBasePath}/workflows/crawl/${ (this.crawl as Crawl).cid }` )} > ${msg("Go to Workflow")} CopyButton.copyToClipboard((this.crawl as Crawl).cid)} > ${msg("Copy Workflow ID")} ` )} CopyButton.copyToClipboard(this.crawl!.tags.join(", "))} ?disabled=${!this.crawl.tags.length} > ${msg("Copy Tags")} ${when( this.isCrawler && !isActive(this.crawl.state), () => html` this.deleteCrawl()} > ${msg("Delete Crawl")} ` )} `; } private renderTitle(title: string) { return html`

${title}

`; } private renderPanel( heading: string | TemplateResult, content: any, classes: any = {} ) { const headingIsTitle = typeof heading === "string"; return html`
${headingIsTitle ? this.renderTitle(heading) : heading}
${content}
`; } private renderReplay() { if (!this.crawl) return; const replaySource = `/api/orgs/${this.crawl.oid}/${ this.crawl.type === "upload" ? "uploads" : "crawls" }/${this.crawlId}/replay.json`; const headers = this.authState?.headers; const config = JSON.stringify({ headers }); const canReplay = this.hasFiles; return html` ${ canReplay ? html`
` : html`

${this.isActive ? msg("No files yet.") : msg("No files to replay.")}

` } `; } private renderOverview() { return html` ${this.crawl ? html` ` : html``} ${when(this.crawl, () => this.crawl!.type === "upload" ? html` ` : html` ${this.crawl!.finished ? html`` : html`${msg("Pending")}`} ${this.crawl!.finished ? html`${RelativeDuration.humanize( new Date(`${this.crawl!.finished}Z`).valueOf() - new Date(`${this.crawl!.started}Z`).valueOf() )}` : html` `} ${this.crawl!.finished ? html`${humanizeExecutionSeconds( this.crawl!.crawlExecSeconds )}` : html`${msg("Pending")}`} ${this.crawl!.manual ? msg( html`Manual start by ${this.crawl!.userName || this.crawl!.userid}` ) : msg(html`Scheduled start`)} ` )} ${this.crawl ? html`${this.crawl.fileSize ? html`${this.crawl.stats ? html`, ${this.numberFormatter.format( +this.crawl.stats.done )} / ${this.numberFormatter.format( +this.crawl.stats.found )} pages` : ""}` : html`${msg("Unknown")}`}` : html``} ${this.renderCrawlChannelVersion()} ${this.crawl ? html`` : html``} `; } private renderCrawlChannelVersion() { if (!this.crawl) { return html``; } const text = capitalize(this.crawl.crawlerChannel || "default") + (this.crawl.image ? ` (${this.crawl.image})` : ""); return html`
${text}
`; } private renderMetadata() { const noneText = html`${msg("None")}`; return html` ${when( this.crawl, () => when( this.crawl!.description?.length, () => html`
${this.crawl?.description}
                
`, () => noneText ), () => html`` )}
${when( this.crawl, () => when( this.crawl!.tags.length, () => this.crawl!.tags.map( (tag) => html`${tag}` ), () => noneText ), () => html`` )} ${when( this.crawl, () => when( this.crawl!.collections.length, () => html`
    ${this.crawl!.collections.map( ({ id, name }) => html`
  • ${name}
  • ` )}
`, () => noneText ), () => html`` )}
`; } private renderFiles() { return html` ${this.hasFiles ? html` ` : html`

${this.isActive ? msg("No files yet.") : msg("No files to download.")}

`} `; } private renderLogs() { return html`
${when(this.logs, () => this.logs!.total ? html` { await this.fetchCrawlLogs({ page: e.detail.page, }); // Scroll to top of list this.scrollIntoView(); }} > ` : html`

${msg("No error logs to display.")}

` )}
`; } private renderConfig() { return html`
${when( this.crawl && this.seeds && (!this.workflowId || this.workflow), () => html` `, this.renderLoading )}
`; } private renderLoading = () => html`
`; /** * Fetch crawl and update internal state */ private async fetchCrawl(): Promise { try { this.crawl = await this.getCrawl(); } catch { this.notify({ message: msg("Sorry, couldn't retrieve crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async fetchSeeds(): Promise { try { this.seeds = await this.getSeeds(); } catch { this.notify({ message: msg( "Sorry, couldn't retrieve all crawl settings at this time." ), variant: "danger", icon: "exclamation-octagon", }); } } private async fetchWorkflow(): Promise { try { this.workflow = await this.getWorkflow(); } catch (e: unknown) { console.debug(e); } } private async getCrawl(): Promise { const apiPath = `/orgs/${this.orgId}/${ this.itemType === "upload" ? "uploads" : "crawls" }/${this.crawlId}/replay.json`; return this.apiFetch(apiPath, this.authState!); } private async getSeeds() { // NOTE Returns first 1000 seeds (backend pagination max) const data = await this.apiFetch>( `/orgs/${this.orgId}/crawls/${this.crawlId}/seeds`, this.authState! ); return data; } private async getWorkflow(): Promise { return this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${this.workflowId}`, this.authState! ); } private async fetchCrawlLogs( params: Partial = {} ): Promise { if (this.itemType !== "crawl") { return; } try { this.logs = await this.getCrawlErrors(params); } catch (e: unknown) { console.debug(e); this.notify({ message: msg("Sorry, couldn't retrieve crawl logs at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async getCrawlErrors(params: Partial) { const page = params.page || this.logs?.page || 1; const pageSize = params.pageSize || this.logs?.pageSize || 50; const data = (await this.apiFetch)>( `/orgs/${this.orgId}/crawls/${this.crawlId}/errors?page=${page}&pageSize=${pageSize}`, this.authState! ); return data; } private async cancel() { if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) { const data = await this.apiFetch<{ success: boolean }>( `/orgs/${this.crawl!.oid}/crawls/${this.crawlId}/cancel`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchCrawl(); } else { this.notify({ message: msg("Sorry, couldn't cancel crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private async stop() { if (window.confirm(msg("Are you sure you want to stop the crawl?"))) { const data = await this.apiFetch<{ success: boolean }>( `/orgs/${this.crawl!.oid}/crawls/${this.crawlId}/stop`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchCrawl(); } else { this.notify({ message: msg("Sorry, couldn't stop crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private openMetadataEditor() { this.openDialogName = "metadata"; } async checkFormValidity(formEl: HTMLFormElement) { await this.updateComplete; return !formEl.querySelector("[data-invalid]"); } private async deleteCrawl() { if ( !window.confirm( msg(str`Are you sure you want to delete crawl of ${this.renderName()}?`) ) ) { return; } try { const _data = await this.apiFetch( `/orgs/${this.crawl!.oid}/${ this.crawl!.type === "crawl" ? "crawls" : "uploads" }/delete`, this.authState!, { method: "POST", body: JSON.stringify({ crawl_ids: [this.crawl!.id], }), } ); this.navTo(this.listUrl); this.notify({ message: msg(`Successfully deleted crawl`), variant: "success", icon: "check2-circle", }); } catch (e: any) { let message = msg( str`Sorry, couldn't delete archived item at this time.` ); if (e.isApiError) { if (e.details == "not_allowed") { message = msg( str`Only org owners can delete other users' archived items.` ); } else if (e.message) { message = e.message; } } this.notify({ message: message, variant: "danger", icon: "exclamation-octagon", }); } } /** Callback when crawl is no longer running */ private _crawlDone() { if (!this.crawl) return; this.fetchCrawlLogs(); this.notify({ message: msg(html`Done crawling ${this.renderName()}.`), variant: "success", icon: "check2-circle", }); if (this.sectionName === "watch") { // Show replay tab this.sectionName = "replay"; } } /** * Enter fullscreen mode * @param id ID of element to fullscreen */ private async _enterFullscreen(id: string) { try { document.getElementById(id)!.requestFullscreen({ // Show browser navigation controls navigationUI: "show", }); } catch (err) { console.error(err); } } }