import type { TemplateResult } from "lit"; import { state, property } from "lit/decorators.js"; import { when } from "lit/directives/when.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { classMap } from "lit/directives/class-map.js"; import { msg, localized, str } from "@lit/localize"; import type { PageChangeEvent } from "../../components/pagination"; import { RelativeDuration } from "../../components/relative-duration"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import { isActive } from "../../utils/crawler"; import { CopyButton } from "../../components/copy-button"; import type { Crawl, Workflow } from "./types"; import { APIPaginatedList } from "../../types/api"; const SECTIONS = [ "overview", "watch", "replay", "files", "logs", "config", "exclusions", ] as const; type SectionName = (typeof SECTIONS)[number]; const LOG_LEVEL_VARIANTS = { error: "danger", } as const; const POLL_INTERVAL_SECONDS = 10; /** * Usage: * ```ts * * ``` */ @localized() export class CrawlDetail extends LiteElement { @property({ type: Object }) authState?: AuthState; // e.g. `/org/${this.orgId}/crawls` @property({ type: String }) crawlsBaseUrl!: string; // e.g. `/org/${this.orgId}/crawls` @property({ type: String }) crawlsAPIBaseUrl?: string; @property({ type: Boolean }) showOrgLink = false; @property({ type: String }) crawlId!: string; @property({ type: Boolean }) isCrawler!: boolean; @state() private crawl?: Crawl; @state() private logs?: APIPaginatedList; @state() private sectionName: SectionName = "overview"; @state() private isSubmittingUpdate: boolean = false; @state() private openDialogName?: "scale" | "metadata" | "exclusions"; @state() private isDialogVisible: boolean = false; // TODO localize private numberFormatter = new Intl.NumberFormat(); private dateFormatter = new Intl.DateTimeFormat(undefined, { year: "numeric", month: "numeric", day: "numeric", }); private get isActive(): boolean | null { if (!this.crawl) return null; return ( this.crawl.state === "running" || this.crawl.state === "starting" || this.crawl.state === "waiting_capacity" || this.crawl.state === "waiting_org_limit" || this.crawl.state === "stopping" ); } private get hasFiles(): boolean | null { if (!this.crawl) return null; if (!this.crawl.resources) return false; return this.crawl.resources.length > 0; } firstUpdated() { if (!this.crawlsBaseUrl) { throw new Error("Crawls base URL not defined"); } this.fetchCrawl(); this.fetchCrawlLogs(); } willUpdate(changedProperties: Map) { const prevId = changedProperties.get("crawlId"); if (prevId && prevId !== this.crawlId) { // Handle update on URL change, e.g. from re-run this.fetchCrawl(); this.fetchCrawlLogs(); } else { const prevCrawl = changedProperties.get("crawl"); if (prevCrawl && this.crawl) { if ( (prevCrawl.state === "running" || prevCrawl.state === "stopping") && !this.isActive ) { this.crawlDone(); } } } } connectedCallback(): void { // Set initial active section based on URL #hash value const hash = window.location.hash.slice(1); if (SECTIONS.includes(hash as any)) { this.sectionName = hash as SectionName; } super.connectedCallback(); } render() { let sectionContent: string | TemplateResult = ""; switch (this.sectionName) { case "replay": sectionContent = this.renderPanel( msg("Replay Crawl"), this.renderReplay(), { "overflow-hidden": true, "rounded-lg": true, border: true, } ); break; case "files": sectionContent = this.renderPanel( msg("Download Files"), this.renderFiles() ); break; case "logs": sectionContent = this.renderPanel(msg("Error Logs"), this.renderLogs()); break; case "config": sectionContent = this.renderPanel(msg("Config"), this.renderConfig(), { "p-4": true, "rounded-lg": true, border: true, }); break; default: sectionContent = html`
${this.renderPanel(msg("Overview"), this.renderOverview(), { "p-4": true, "rounded-lg": true, border: true, })}
${this.renderPanel( html` ${msg("Metadata")} ${when( this.isCrawler, () => html` ` )} `, this.renderMetadata(), { "p-4": true, "rounded-lg": true, border: true, } )}
`; break; } // TODO abstract into breadcrumbs const isWorkflowArtifact = this.crawlsBaseUrl.includes("/workflows/"); return html`
${isWorkflowArtifact ? msg("Back to Crawl Workflow") : msg("Back to Finished Crawls")}
${this.renderHeader()}

${this.renderNav()}
${sectionContent}
(this.openDialogName = undefined)} @updated=${() => this.fetchCrawl()} > `; } private renderName() { if (!this.crawl) return html``; if (this.crawl.name) return this.crawl.name; if (!this.crawl.firstSeed) return this.crawl.id; const remainder = this.crawl.seedCount - 1; let crawlName: any = html`${this.crawl.firstSeed}`; if (remainder) { if (remainder === 1) { crawlName = msg( html`${this.crawl.firstSeed} +${remainder} URL` ); } else { crawlName = msg( html`${this.crawl.firstSeed} +${remainder} URLs` ); } } return crawlName; } private renderNav() { const renderNavItem = ({ section, label, iconLibrary, icon, }: { section: SectionName; label: any; iconLibrary: "app" | "default"; icon: string; }) => { const isActive = section === this.sectionName; const baseUrl = window.location.pathname.split("#")[0]; return html` `; }; return html` `; } private renderHeader() { return html`

${this.renderName()}

${this.isActive ? html` ${msg("Stop")} ${msg("Cancel")} ` : ""} ${this.crawl && this.isCrawler ? this.renderMenu() : ""}
`; } private renderMenu() { if (!this.crawl) return; const crawlId = this.crawl.id; const closeDropdown = (e: any) => { e.target.closest("sl-dropdown").hide(); }; return html` ${this.isActive ? html`` : msg("Actions")} ${when( this.isCrawler, () => html` { this.openMetadataEditor(); }} > ${msg("Edit Metadata")} ` )} this.navTo( `/orgs/${this.crawl!.oid}/workflows/crawl/${this.crawl!.cid}` )} > ${msg("Go to Workflow")} CopyButton.copyToClipboard(this.crawl!.cid)} > ${msg("Copy Workflow ID")} CopyButton.copyToClipboard(this.crawl!.tags.join(", "))} ?disabled=${!this.crawl.tags.length} > ${msg("Copy Tags")} ${when( this.isCrawler && !isActive(this.crawl.state), () => html` this.deleteCrawl()} > ${msg("Delete Crawl")} ` )} `; } private renderPanel(title: any, content: any, classes: any = {}) { return html`

${title}

${content}
`; } // renders the info bar, currently disabled private renderSummary() { return html`
${msg("Status")}
${this.crawl ? html` ` : html``}
${msg("Pages Crawled")}
${this.crawl?.stats ? html` ${this.numberFormatter.format(+this.crawl.stats.done)} / ${this.numberFormatter.format(+this.crawl.stats.found)} ` : this.crawl ? html` ${msg("Unknown")} ` : html``}
${msg("Run Duration")}
${this.crawl ? html` ${this.crawl.finished ? html`${RelativeDuration.humanize( new Date(`${this.crawl.finished}Z`).valueOf() - new Date(`${this.crawl.started}Z`).valueOf() )}` : html` `} ` : html``}
${msg("Crawler Instances")}
${this.crawl ? this.crawl?.scale : html``}
`; } private renderReplay() { //const replaySource = `/api/orgs/${this.crawl?.oid}/crawls/${this.crawlId}/replay.json?auth_bearer=${bearer}`; const replaySource = `/api/orgs/${this.crawl?.oid}/crawls/${this.crawlId}/replay.json`; const headers = this.authState?.headers; const config = JSON.stringify({headers}); const canReplay = replaySource && this.hasFiles; return html` ${ canReplay ? html`
` : html`

${this.isActive ? msg("No files yet.") : msg("No files to replay.")}

` } `; } private renderOverview() { return html` ${this.crawl ? html` ` : html``} ${this.crawl ? html` ` : html``} ${this.crawl ? html` ${this.crawl.finished ? html`` : html`${msg("Pending")}`} ` : html``} ${this.crawl ? html` ${this.crawl.finished ? html`${RelativeDuration.humanize( new Date(`${this.crawl.finished}Z`).valueOf() - new Date(`${this.crawl.started}Z`).valueOf() )}` : html` `} ` : html``} ${this.crawl?.stats ? html` , ${this.numberFormatter.format(+this.crawl.stats.done)} / ${this.numberFormatter.format(+this.crawl.stats.found)} pages ` : this.crawl ? html` ${msg("Unknown")} ` : html``} ${this.crawl ? html` ${this.crawl.manual ? msg( html`Manual start by ${this.crawl?.userName || this.crawl?.userid}` ) : msg(html`Scheduled start`)} ` : html``} ${this.crawl ? html` ${this.crawl.id} ` : html``} ${this.showOrgLink ? html` ${this.crawl ? html` ${msg("View Organization")} ` : html``} ` : ""} `; } private renderMetadata() { const noneText = html`${msg("None")}`; return html` ${when( this.crawl, () => when( this.crawl!.notes?.length, () => html`
${this.crawl?.notes}
                
`, () => noneText ), () => html`` )}
${when( this.crawl, () => when( this.crawl!.tags.length, () => this.crawl!.tags.map( (tag) => html`${tag}` ), () => noneText ), () => html`` )}
`; } private renderFiles() { return html` ${this.hasFiles ? html` ` : html`

${this.isActive ? msg("No files yet.") : msg("No files to download.")}

`} `; } private renderLogs() { if (!this.logs) { return html`
`; } if (!this.logs.total) { return html`

${msg("No error logs to display.")}

`; } return html` { await this.fetchCrawlLogs({ page: e.detail.page, }); // Scroll to top of list this.scrollIntoView(); }} > `; } private renderConfig() { if (!this.crawl?.config) return ""; return html` `; } /** * Fetch crawl and update internal state */ private async fetchCrawl(): Promise { try { this.crawl = await this.getCrawl(); } catch { this.notify({ message: msg("Sorry, couldn't retrieve crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async getCrawl(): Promise { const data: Crawl = await this.apiFetch( `${this.crawlsAPIBaseUrl || this.crawlsBaseUrl}/${ this.crawlId }/replay.json`, this.authState! ); return data; } private async fetchCrawlLogs( params: Partial = {} ): Promise { try { this.logs = await this.getCrawlLogs(params); } catch { this.notify({ message: msg("Sorry, couldn't retrieve crawl logs at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async getCrawlLogs( params: Partial ): Promise { const page = params.page || this.logs?.page || 1; const pageSize = params.pageSize || this.logs?.pageSize || 50; const data: APIPaginatedList = await this.apiFetch( `${this.crawlsAPIBaseUrl || this.crawlsBaseUrl}/${ this.crawlId }/errors?page=${page}&pageSize=${pageSize}`, this.authState! ); return data; } private async cancel() { if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) { const data = await this.apiFetch( `/orgs/${this.crawl!.oid}/crawls/${this.crawlId}/cancel`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchCrawl(); } else { this.notify({ message: msg("Sorry, couldn't cancel crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private async stop() { if (window.confirm(msg("Are you sure you want to stop the crawl?"))) { const data = await this.apiFetch( `/orgs/${this.crawl!.oid}/crawls/${this.crawlId}/stop`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchCrawl(); } else { this.notify({ message: msg("Sorry, couldn't stop crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private openMetadataEditor() { this.openDialogName = "metadata"; } async checkFormValidity(formEl: HTMLFormElement) { await this.updateComplete; return !formEl.querySelector("[data-invalid]"); } private async runNow() { if (!this.crawl) return; try { const data = await this.apiFetch( `/orgs/${this.crawl.oid}/crawlconfigs/${this.crawl.cid}/run`, this.authState!, { method: "POST", } ); if (data.started) { this.navTo(`/orgs/${this.crawl.oid}/artifacts/crawl/${data.started}`); } this.notify({ message: msg( html`Started crawl from ${this.renderName()}.` ), variant: "success", icon: "check2-circle", duration: 8000, }); } catch (e: any) { if (e.isApiError && e.message === "crawl_already_running") { this.notify({ message: msg( html`Crawl of ${this.renderName()} is already running.` ), variant: "warning", icon: "exclamation-triangle", }); } else { this.notify({ message: msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private async deleteCrawl() { if ( !window.confirm( msg(str`Are you sure you want to delete crawl of ${this.renderName()}?`) ) ) { return; } try { const data = await this.apiFetch( `/orgs/${this.crawl!.oid}/crawls/delete`, this.authState!, { method: "POST", body: JSON.stringify({ crawl_ids: [this.crawl!.id], }), } ); this.navTo(this.crawlsBaseUrl); this.notify({ message: msg(`Successfully deleted crawl`), variant: "success", icon: "check2-circle", }); } catch (e: any) { this.notify({ message: (e.isApiError && e.message) || msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } /** Callback when crawl is no longer running */ private crawlDone() { if (!this.crawl) return; this.fetchCrawlLogs(); this.notify({ message: msg(html`Done crawling ${this.renderName()}.`), variant: "success", icon: "check2-circle", }); if (this.sectionName === "watch") { // Show replay tab this.sectionName = "replay"; } } /** * Enter fullscreen mode * @param id ID of element to fullscreen */ private async enterFullscreen(id: string) { try { document.getElementById(id)!.requestFullscreen({ // Show browser navigation controls navigationUI: "show", }); } catch (err) { console.error(err); } } } customElements.define("btrix-crawl-detail", CrawlDetail);