import type { TemplateResult } from "lit"; import { state, property } from "lit/decorators.js"; import { when } from "lit/directives/when.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { msg, localized, str } from "@lit/localize"; import { RelativeDuration } from "../../components/relative-duration"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import { CopyButton } from "../../components/copy-button"; import type { Crawl, Workflow } from "./types"; const SECTIONS = [ "overview", "watch", "replay", "files", "logs", "config", "exclusions", ] as const; type SectionName = (typeof SECTIONS)[number]; const POLL_INTERVAL_SECONDS = 10; /** * Usage: * ```ts * * ``` */ @localized() export class CrawlDetail extends LiteElement { @property({ type: Object }) authState?: AuthState; // e.g. `/org/${this.orgId}/crawls` @property({ type: String }) crawlsBaseUrl!: string; // e.g. `/org/${this.orgId}/crawls` @property({ type: String }) crawlsAPIBaseUrl?: string; @property({ type: Boolean }) showOrgLink = false; @property({ type: String }) crawlId!: string; @property({ type: Boolean }) isCrawler!: boolean; @state() private crawl?: Crawl; @state() private sectionName: SectionName = "overview"; @state() private isSubmittingUpdate: boolean = false; @state() private openDialogName?: "scale" | "metadata" | "exclusions"; @state() private isDialogVisible: boolean = false; private timerId?: number; // TODO localize private numberFormatter = new Intl.NumberFormat(); private get isActive(): boolean | null { if (!this.crawl) return null; return ( this.crawl.state === "running" || this.crawl.state === "starting" || this.crawl.state === "stopping" ); } private get hasFiles(): boolean | null { if (!this.crawl) return null; if (!this.crawl.resources) return false; return this.crawl.resources.length > 0; } firstUpdated() { if (!this.crawlsBaseUrl) { throw new Error("Crawls base URL not defined"); } this.fetchCrawl(); } willUpdate(changedProperties: Map) { const prevId = changedProperties.get("crawlId"); if (prevId && prevId !== this.crawlId) { // Handle update on URL change, e.g. from re-run this.stopPollTimer(); this.fetchCrawl(); } else { const prevCrawl = changedProperties.get("crawl"); if (prevCrawl && this.crawl) { if ( (prevCrawl.state === "running" || prevCrawl.state === "stopping") && !this.isActive ) { this.crawlDone(); } } } } connectedCallback(): void { // Set initial active section based on URL #hash value const hash = window.location.hash.slice(1); if (SECTIONS.includes(hash as any)) { this.sectionName = hash as SectionName; } super.connectedCallback(); } disconnectedCallback(): void { this.stopPollTimer(); super.disconnectedCallback(); } render() { let sectionContent: string | TemplateResult = ""; switch (this.sectionName) { case "exclusions": case "watch": { if (this.crawl) { const isRunning = this.crawl.state === "running"; sectionContent = this.renderPanel( html`${msg("Watch Crawl")} { this.openDialogName = "scale"; this.isDialogVisible = true; }} > ${msg("Crawler Instances")} `, this.renderWatch() ); } else { // TODO loading indicator? return ""; } break; } case "replay": sectionContent = this.renderPanel( msg("Replay Crawl"), this.renderReplay() ); break; case "files": sectionContent = this.renderPanel( msg("Download Files"), this.renderFiles() ); break; case "logs": sectionContent = this.renderPanel(msg("Logs"), this.renderLogs()); break; case "config": sectionContent = this.renderPanel(msg("Config"), this.renderConfig()); break; default: sectionContent = html`
${this.renderPanel(msg("Overview"), this.renderOverview())}
${this.renderPanel( html` ${msg("Metadata")} ${when( this.isCrawler, () => html` ` )} `, this.renderMetadata() )}
`; break; } return html`
${msg("Back to Crawls")}
${this.renderHeader()}
${this.renderSummary()}
${this.renderNav()}
${sectionContent}
(this.openDialogName = undefined)} @updated=${() => this.fetchCrawl()} > `; } private renderName() { if (!this.crawl) return html``; if (this.crawl.name) return this.crawl.name; if (!this.crawl.firstSeed) return this.crawl.id; const remainder = this.crawl.seedCount - 1; let crawlName: any = html`${this.crawl.firstSeed}`; if (remainder) { if (remainder === 1) { crawlName = msg( html`${this.crawl.firstSeed} +${remainder} URL` ); } else { crawlName = msg( html`${this.crawl.firstSeed} +${remainder} URLs` ); } } return crawlName; } private renderNav() { const renderNavItem = ({ section, label, iconLibrary, icon, }: { section: SectionName; label: any; iconLibrary: "app" | "default"; icon: string; }) => { const isActive = section === this.sectionName; return html` `; }; return html` `; } private renderHeader() { return html`

${this.renderName()}

${this.isActive ? html` ${msg("Stop")} ${msg("Cancel")} ` : ""} ${this.crawl && this.isCrawler ? this.renderMenu() : ""}
`; } private renderMenu() { if (!this.crawl) return; const crawlId = this.crawl.id; const closeDropdown = (e: any) => { e.target.closest("sl-dropdown").hide(); }; return html` ${this.isActive ? html`` : msg("Actions")} `; } private renderPanel(title: any, content: any) { return html`

${title}

${content}
`; } private renderSummary() { return html`
${msg("Status")}
${this.crawl ? html` ` : html``}
${msg("Pages Crawled")}
${this.crawl?.stats ? html` ${this.numberFormatter.format(+this.crawl.stats.done)} / ${this.numberFormatter.format(+this.crawl.stats.found)} ` : this.crawl ? html` ${msg("Unknown")} ` : html``}
${msg("Run Duration")}
${this.crawl ? html` ${this.crawl.finished ? html`${RelativeDuration.humanize( new Date(`${this.crawl.finished}Z`).valueOf() - new Date(`${this.crawl.started}Z`).valueOf() )}` : html` `} ` : html``}
${msg("Crawler Instances")}
${this.crawl ? this.crawl?.scale : html``}
`; } private renderWatch() { if (!this.authState || !this.crawl) return ""; const isStarting = this.crawl.state === "starting"; const isRunning = this.crawl.state === "running"; const isStopping = this.crawl.state === "stopping"; const authToken = this.authState.headers.Authorization.split(" ")[1]; return html` ${isStarting ? html`

${msg("Crawl starting...")}

` : this.isActive ? html` ${isStopping ? html`
${msg("Crawl stopping...")}
` : ""} ` : this.renderInactiveCrawlMessage()} ${when( isRunning, () => html`
${this.renderExclusions()}
(this.openDialogName = undefined)} @sl-show=${() => (this.isDialogVisible = true)} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${this.isDialogVisible ? this.renderEditScale() : ""} ` )} `; } private renderExclusions() { return html`

${msg("Crawl URLs")}

{ this.openDialogName = "exclusions"; this.isDialogVisible = true; }} > ${msg("Edit Exclusions")}
${when( this.crawl, () => html` ` )} (this.openDialogName = undefined)} @sl-show=${() => (this.isDialogVisible = true)} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${this.isDialogVisible ? html`` : ""}
(this.openDialogName = undefined)} >${msg("Done Editing")}
`; } private renderReplay() { const bearer = this.authState?.headers?.Authorization?.split(" ", 2)[1]; // for now, just use the first file until multi-wacz support is fully implemented const replaySource = `/api/orgs/${this.crawl?.oid}/crawls/${this.crawlId}/replay.json?auth_bearer=${bearer}`; //const replaySource = this.crawl?.resources?.[0]?.path; const canReplay = replaySource && this.hasFiles; return html` ${ canReplay ? html`
` : html`

${this.isActive ? msg("No files yet.") : msg("No files to replay.")}

` } `; } private renderOverview() { return html` ${this.crawl ? html` ` : html``} ${this.crawl ? html` ${this.crawl.finished ? html`` : html`${msg("Pending")}`} ` : html``} ${this.crawl ? html` ${this.crawl.manual ? msg( html`Manual start by ${this.crawl?.userName || this.crawl?.userid}` ) : msg(html`Scheduled run`)} ` : html``} ${this.crawl ? html` ${this.crawl.id} ` : html``} ${this.showOrgLink ? html` ${this.crawl ? html` ${msg("View Organization")} ` : html``} ` : ""} `; } private renderMetadata() { const noneText = html`${msg("None")}`; return html` ${when( this.crawl, () => when( this.crawl!.notes?.length, () => html`
${this.crawl?.notes}
                
`, () => noneText ), () => html`` )}
${when( this.crawl, () => when( this.crawl!.tags.length, () => this.crawl!.tags.map( (tag) => html`${tag}` ), () => noneText ), () => html`` )}
`; } private renderFiles() { return html` ${this.hasFiles ? html` ` : html`

${this.isActive ? msg("No files yet.") : msg("No files to download.")}

`} `; } private renderLogs() { return html`TODO`; } private renderConfig() { if (!this.crawl?.config) return ""; return html` `; } private renderEditScale() { if (!this.crawl) return; const scaleOptions = [ { value: 1, label: "1", }, { value: 2, label: "2", }, { value: 3, label: "3", }, ]; return html`
${scaleOptions.map( ({ value, label }) => html` this.scale(value)} ?disabled=${this.isSubmittingUpdate} >${label} ` )}
(this.openDialogName = undefined)} >${msg("Cancel")}
`; } private renderInactiveCrawlMessage() { return html`

${msg("Crawl is not running.")} ${this.hasFiles ? html` (this.sectionName = "replay")} >View replay` : ""}

`; } /** * Fetch crawl and update internal state */ private async fetchCrawl(): Promise { try { this.crawl = await this.getCrawl(); if (this.isActive) { // Restart timer for next poll this.stopPollTimer(); this.timerId = window.setTimeout(() => { this.fetchCrawl(); }, 1000 * POLL_INTERVAL_SECONDS); } else { this.stopPollTimer(); } } catch { this.notify({ message: msg("Sorry, couldn't retrieve crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async getCrawl(): Promise { const data: Crawl = await this.apiFetch( `${this.crawlsAPIBaseUrl || this.crawlsBaseUrl}/${ this.crawlId }/replay.json`, this.authState! ); return data; } private async cancel() { if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) { const data = await this.apiFetch( `/orgs/${this.crawl!.oid}/crawls/${this.crawlId}/cancel`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchCrawl(); } else { this.notify({ message: msg("Sorry, couldn't cancel crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private async stop() { if (window.confirm(msg("Are you sure you want to stop the crawl?"))) { const data = await this.apiFetch( `/orgs/${this.crawl!.oid}/crawls/${this.crawlId}/stop`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchCrawl(); } else { this.notify({ message: msg("Sorry, couldn't stop crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private openMetadataEditor() { this.openDialogName = "metadata"; } async checkFormValidity(formEl: HTMLFormElement) { await this.updateComplete; return !formEl.querySelector("[data-invalid]"); } private async scale(value: Crawl["scale"]) { this.isSubmittingUpdate = true; try { const data = await this.apiFetch( `/orgs/${this.crawl!.oid}/crawls/${this.crawlId}/scale`, this.authState!, { method: "POST", body: JSON.stringify({ scale: +value }), } ); if (data.scaled) { this.crawl!.scale = data.scaled; this.notify({ message: msg("Updated crawl scale."), variant: "success", icon: "check2-circle", }); } else { throw new Error("unhandled API response"); } this.openDialogName = undefined; this.isDialogVisible = false; } catch { this.notify({ message: msg("Sorry, couldn't change crawl scale at this time."), variant: "danger", icon: "exclamation-octagon", }); } this.isSubmittingUpdate = false; } private async runNow() { if (!this.crawl) return; try { const data = await this.apiFetch( `/orgs/${this.crawl.oid}/crawlconfigs/${this.crawl.cid}/run`, this.authState!, { method: "POST", } ); if (data.started) { this.navTo(`/orgs/${this.crawl.oid}/crawls/crawl/${data.started}`); } this.notify({ message: msg( html`Started crawl from ${this.renderName()}.` ), variant: "success", icon: "check2-circle", duration: 8000, }); } catch (e: any) { if (e.isApiError && e.message === "crawl_already_running") { this.notify({ message: msg( html`Crawl of ${this.renderName()} is already running.` ), variant: "warning", icon: "exclamation-triangle", }); } else { this.notify({ message: msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private async deleteCrawl() { if ( !window.confirm( msg(str`Are you sure you want to delete crawl of ${this.renderName()}?`) ) ) { return; } try { const data = await this.apiFetch( `/orgs/${this.crawl!.oid}/crawls/delete`, this.authState!, { method: "POST", body: JSON.stringify({ crawl_ids: [this.crawl!.id], }), } ); this.navTo(this.crawlsBaseUrl); this.notify({ message: msg(`Successfully deleted crawl`), variant: "success", icon: "check2-circle", }); } catch (e: any) { this.notify({ message: (e.isApiError && e.message) || msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private handleExclusionChange(e: CustomEvent) { this.fetchCrawl(); } private stopPollTimer() { window.clearTimeout(this.timerId); } /** Callback when crawl is no longer running */ private crawlDone() { if (!this.crawl) return; this.notify({ message: msg(html`Done crawling ${this.renderName()}.`), variant: "success", icon: "check2-circle", }); if (this.sectionName === "watch") { // Show replay tab this.sectionName = "replay"; } } /** * Enter fullscreen mode * @param id ID of element to fullscreen */ private async enterFullscreen(id: string) { try { document.getElementById(id)!.requestFullscreen({ // Show browser navigation controls navigationUI: "show", }); } catch (err) { console.error(err); } } } customElements.define("btrix-crawl-detail", CrawlDetail);