import type { HTMLTemplateResult, TemplateResult } from "lit"; import { state, property } from "lit/decorators.js"; import { when } from "lit/directives/when.js"; import { until } from "lit/directives/until.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { msg, localized, str } from "@lit/localize"; import queryString from "query-string"; import { CopyButton } from "../../components/copy-button"; import { CrawlStatus } from "../../components/crawl-status"; import { RelativeDuration } from "../../components/relative-duration"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import type { Crawl, CrawlState, Workflow, WorkflowParams, JobType, } from "./types"; import { humanizeSchedule, humanizeNextDate } from "../../utils/cron"; import { APIPaginatedList } from "../../types/api"; import { inactiveCrawlStates, isActive } from "../../utils/crawler"; import { SlSelect } from "@shoelace-style/shoelace"; import { DASHBOARD_ROUTE } from "../../routes"; const SECTIONS = ["artifacts", "watch", "settings"] as const; type Tab = (typeof SECTIONS)[number]; const DEFAULT_SECTION: Tab = "artifacts"; const POLL_INTERVAL_SECONDS = 10; const ABORT_REASON_CANCLED = "canceled"; /** * Usage: * ```ts * * ``` */ @localized() export class WorkflowDetail extends LiteElement { @property({ type: Object }) authState!: AuthState; @property({ type: String }) orgId!: string; @property({ type: String }) workflowId!: string; @property({ type: Boolean }) isEditing: boolean = false; @property({ type: Boolean }) isCrawler!: boolean; @property({ type: String }) openDialogName?: "scale" | "exclusions" | "cancel" | "stop"; @state() private workflow?: Workflow; @state() private crawls?: APIPaginatedList; // Only inactive crawls @state() private lastCrawlId: Workflow["lastCrawlId"] = null; @state() private lastCrawlStartTime: Workflow["lastCrawlStartTime"] = null; @state() private lastCrawlStats?: Crawl["stats"]; @state() private activePanel: Tab = SECTIONS[0]; @state() private isLoading: boolean = false; @state() private isSubmittingUpdate: boolean = false; @state() private isDialogVisible: boolean = false; @state() private isCancelingOrStoppingCrawl: boolean = false; @state() private filterBy: Partial> = {}; // TODO localize private numberFormatter = new Intl.NumberFormat(undefined, { // notation: "compact", }); private dateFormatter = new Intl.DateTimeFormat(undefined, { year: "numeric", month: "numeric", day: "numeric", }); private timerId?: number; private isPanelHeaderVisible?: boolean; private getWorkflowPromise?: Promise; private readonly jobTypeLabels: Record = { "url-list": msg("URL List"), "seed-crawl": msg("Seeded Crawl"), custom: msg("Custom"), }; private readonly tabLabels: Record = { artifacts: msg("Crawls"), watch: msg("Watch Crawl"), settings: msg("Workflow Settings"), }; connectedCallback(): void { // Set initial active section and dialog based on URL #hash value this.getActivePanelFromHash(); super.connectedCallback(); window.addEventListener("hashchange", this.getActivePanelFromHash); } disconnectedCallback(): void { this.stopPoll(); super.disconnectedCallback(); window.removeEventListener("hashchange", this.getActivePanelFromHash); } firstUpdated() { if ( this.openDialogName && (this.openDialogName === "scale" || this.openDialogName === "exclusions") ) { this.showDialog(); } } willUpdate(changedProperties: Map) { if ( (changedProperties.has("workflowId") && this.workflowId) || (changedProperties.get("isEditing") === true && this.isEditing === false) ) { this.fetchWorkflow(); } if (changedProperties.has("isEditing") && this.isEditing) { this.stopPoll(); } if ( changedProperties.get("lastCrawlId") && !this.lastCrawlId && this.activePanel === "watch" ) { this.handleCrawlRunEnd(); } if ( !this.isEditing && changedProperties.has("activePanel") && this.activePanel ) { if (!this.isPanelHeaderVisible) { // Scroll panel header into view this.querySelector("btrix-tab-list")?.scrollIntoView({ behavior: "smooth", }); } if (this.activePanel === "artifacts") { this.fetchCrawls(); } } } private getActivePanelFromHash = async () => { await this.updateComplete; if (this.isEditing) return; const hashValue = window.location.hash.slice(1); if (SECTIONS.includes(hashValue as any)) { this.activePanel = hashValue as Tab; } else { this.goToTab(DEFAULT_SECTION, { replace: true }); } }; private goToTab(tab: Tab, { replace = false } = {}) { const path = `${window.location.href.split("#")[0]}#${tab}`; if (replace) { window.history.replaceState(null, "", path); } else { window.history.pushState(null, "", path); } this.activePanel = tab; } private async handleCrawlRunEnd() { this.goToTab("artifacts", { replace: true }); await this.fetchWorkflow(); let notifyOpts = { message: msg("Crawl finished."), variant: "info", icon: "info-circle", } as any; // TODO consolidate with `CrawlStatus.getContent` switch (this.workflow!.lastCrawlState) { case "complete": notifyOpts = { message: msg("Crawl complete."), variant: "success", icon: "check-circle", }; break; case "canceled": notifyOpts = { message: msg("Crawl canceled."), variant: "danger", icon: "x-octagon", }; break; case "failed": notifyOpts = { message: msg("Crawl failed."), variant: "danger", icon: "exclamation-triangle", }; break; default: break; } this.notify({ ...notifyOpts, duration: 8000, }); } private async fetchWorkflow() { this.stopPoll(); this.isLoading = true; try { this.getWorkflowPromise = this.getWorkflow(); this.workflow = await this.getWorkflowPromise; this.lastCrawlId = this.workflow.lastCrawlId; this.lastCrawlStartTime = this.workflow.lastCrawlStartTime; if (this.lastCrawlId) { this.fetchCurrentCrawlStats(); } } catch (e: any) { this.notify({ message: e.statusCode === 404 ? msg("Workflow not found.") : msg("Sorry, couldn't retrieve Workflow at this time."), variant: "danger", icon: "exclamation-octagon", }); } this.isLoading = false; if (!this.isEditing) { // Restart timer for next poll this.timerId = window.setTimeout(() => { this.fetchWorkflow(); }, 1000 * POLL_INTERVAL_SECONDS); } } render() { if (this.isEditing && this.isCrawler) { return html`
${when(this.workflow, this.renderEditor)}
`; } return html`
${this.renderHeader()}

${this.renderName()} ${when( this.workflow?.inactive, () => html` ${msg("Inactive")} ` )}

${when( this.isCrawler && this.workflow && !this.workflow.inactive, this.renderActions )}
${this.renderDetails()}
${when( this.workflow, this.renderTabList, () => html`
` )}
(this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${msg( "Pages crawled so far will be saved and marked as incomplete. Are you sure you want to stop crawling?" )}
(this.openDialogName = undefined)} >Keep Crawling { await this.stop(); this.openDialogName = undefined; }} >Stop Crawling
(this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${msg( "Canceling will discard all pages crawled. Are you sure you want to discard them?" )}
(this.openDialogName = undefined)} >Keep Crawling { await this.cancel(); this.openDialogName = undefined; }} >Cancel & Discard Crawl
`; } private renderHeader(workflowId?: string) { return html` `; } private renderTabList = () => html` (this.isPanelHeaderVisible = detail.entry.isIntersecting)} >
${this.renderPanelHeader()}
${this.renderTab("artifacts")} ${this.renderTab("watch", { disabled: !this.lastCrawlId })} ${this.renderTab("settings")} ${this.renderArtifacts()} ${until( this.getWorkflowPromise?.then( () => html` ${when(this.activePanel === "watch", () => this.workflow?.isCrawlRunning ? html`
${this.renderCurrentCrawl()}
${this.renderWatchCrawl()}` : this.renderInactiveWatchCrawl() )} ` ) )}
${this.renderSettings()}
`; private renderPanelHeader() { if (!this.activePanel) return; if (this.activePanel === "artifacts") { return html`

${this.tabLabels[this.activePanel]} ${when( this.crawls, () => html` (${this.crawls!.total.toLocaleString()}${this.workflow ?.isCrawlRunning ? html` + 1` : ""}) ` )}

`; } if (this.activePanel === "settings") { return html`

${this.tabLabels[this.activePanel]}

this.navTo( `/orgs/${this.workflow?.oid}/workflows/crawl/${this.workflow?.id}?edit` )} > `; } if (this.activePanel === "watch") { return html`

${this.tabLabels[this.activePanel]}

(this.openDialogName = "scale")} > ${msg("Edit Instances")} `; } return html`

${this.tabLabels[this.activePanel]}

`; } private renderTab(tabName: Tab, { disabled = false } = {}) { const isActive = tabName === this.activePanel; let className = "text-neutral-600 hover:bg-neutral-50"; if (isActive) { className = "text-blue-600 bg-blue-50 shadow-sm"; } else if (disabled) { className = "text-neutral-300 cursor-not-allowed"; } return html` { if (disabled) e.preventDefault(); }} > ${this.tabLabels[tabName]} `; } private renderEditor = () => html` ${this.renderHeader(this.workflow!.id)}

${this.renderName()}

${when( !this.isLoading, () => html` this.navTo( `/orgs/${this.orgId}/workflows/crawl/${this.workflow!.id}` )} > ` )} `; private renderActions = () => { if (!this.workflow) return; const workflow = this.workflow; return html` ${when( this.workflow?.isCrawlRunning, () => html` (this.openDialogName = "stop")} ?disabled=${!this.lastCrawlId || this.isCancelingOrStoppingCrawl || this.workflow?.lastCrawlStopping} > ${msg("Stop")} (this.openDialogName = "cancel")} ?disabled=${!this.lastCrawlId || this.isCancelingOrStoppingCrawl} > ${msg("Cancel")} `, () => html` this.runNow()} > ${msg("Run Crawl")} ` )} ${msg("Actions")} ${when( this.workflow?.isCrawlRunning, // HACK shoelace doesn't current have a way to override non-hover // color without resetting the --sl-color-neutral-700 variable () => html` (this.openDialogName = "stop")} ?disabled=${workflow.lastCrawlStopping || this.isCancelingOrStoppingCrawl} > ${msg("Stop Crawl")} (this.openDialogName = "cancel")} > ${msg("Cancel & Discard Crawl")} `, () => html` this.runNow()} > ${msg("Run Crawl")} ` )} ${when( workflow.isCrawlRunning, () => html` (this.openDialogName = "scale")}> ${msg("Edit Crawler Instances")} (this.openDialogName = "exclusions")} > ${msg("Edit Exclusions")} ` )} this.navTo( `/orgs/${workflow.oid}/workflows/crawl/${workflow.id}?edit` )} > ${msg("Edit Workflow Settings")} CopyButton.copyToClipboard(workflow.tags.join(", "))} ?disabled=${!workflow.tags.length} > ${msg("Copy Tags")} this.duplicateConfig()}> ${msg("Duplicate Workflow")} ${when(!this.lastCrawlId, () => { const shouldDeactivate = workflow.crawlCount && !workflow.inactive; return html` shouldDeactivate ? this.deactivate() : this.delete()} > ${shouldDeactivate ? msg("Deactivate Workflow") : msg("Delete Workflow")} `; })} `; }; private renderDetails() { return html`
${this.renderDetailItem( msg("Status"), () => html` ` )} ${this.renderDetailItem( msg("Total Size"), () => html` ` )} ${this.renderDetailItem(msg("Schedule"), () => this.workflow!.schedule ? html`
${humanizeSchedule(this.workflow!.schedule, { length: "short", })}
` : html`${msg("No Schedule")}` )} ${this.renderDetailItem( msg("Created By"), () => msg( str`${ this.workflow!.createdByName } on ${this.dateFormatter.format( new Date(`${this.workflow!.created}Z`) )}` ), true )}
`; } private renderDetailItem( label: string | TemplateResult, renderContent: () => any, isLast = false ) { return html` ${when( this.workflow, renderContent, () => html`` )} ${when(!isLast, () => html`
`)} `; } private renderName() { if (!this.workflow) return ""; if (this.workflow.name) return this.workflow.name; const { config } = this.workflow; const firstSeed = config.seeds[0]; let firstSeedURL = firstSeed.url; if (config.seeds.length === 1) { return firstSeedURL; } const remainderCount = config.seeds.length - 1; if (remainderCount === 1) { return msg( html`${firstSeedURL} +${remainderCount} URL` ); } return msg( html`${firstSeedURL} +${remainderCount} URLs` ); } private renderArtifacts() { return html`
${msg("View:")}
{ const value = (e.target as SlSelect).value as CrawlState[]; await this.updateComplete; this.filterBy = { ...this.filterBy, state: value, }; this.fetchCrawls(); }} > ${inactiveCrawlStates.map(this.renderStatusMenuItem)}
${when( this.workflow?.isCrawlRunning, () => html`
${msg( html`Crawl is currently running. Watch Crawl Progress` )}
` )} ${msg("Start Time")} ${when( this.crawls, () => this.crawls!.items.map( (crawl: Crawl) => html` this.deleteCrawl(crawl)} > ${msg("Delete Crawl")} ` ), () => html`
` )}
${when( this.crawls && !this.crawls.items.length, () => html`

${this.crawls?.total ? msg("No matching crawls found.") : msg("No crawls yet.")}

` )}
`; } private renderStatusMenuItem = (state: CrawlState) => { const { icon, label } = CrawlStatus.getContent(state); return html`${icon}${label}`; }; private renderCurrentCrawl = () => { const skeleton = html``; return html`
${this.renderDetailItem(msg("Pages Crawled"), () => this.lastCrawlStats ? msg( str`${this.numberFormatter.format( +(this.lastCrawlStats.done || 0) )} / ${this.numberFormatter.format( +(this.lastCrawlStats.found || 0) )}` ) : html`` )} ${this.renderDetailItem(msg("Run Duration"), () => this.lastCrawlStartTime ? RelativeDuration.humanize( new Date().valueOf() - new Date(`${this.lastCrawlStartTime}Z`).valueOf() ) : skeleton )} ${this.renderDetailItem(msg("Crawl Size"), () => this.workflow ? html`` : skeleton )} ${this.renderDetailItem( msg("Crawler Instances"), () => (this.workflow ? this.workflow.scale : skeleton), true )}
`; }; private renderWatchCrawl = () => { if (!this.authState || !this.workflow?.lastCrawlState) return ""; let waitingMsg = null; switch (this.workflow.lastCrawlState) { case "starting": waitingMsg = msg("Crawl starting..."); break; case "waiting_capacity": waitingMsg = msg( "Crawl waiting for available resources before it can start..." ); break; case "waiting_org_limit": waitingMsg = msg( "Crawl waiting for others to finish, concurrent limit per Organization reached..." ); break; } const isRunning = this.workflow.lastCrawlState === "running"; const isStopping = this.workflow.lastCrawlStopping; const authToken = this.authState.headers.Authorization.split(" ")[1]; return html` ${waitingMsg ? html`

${waitingMsg}

` : isActive(this.workflow.lastCrawlState) ? html` ${isStopping ? html`
${msg("Crawl stopping...")}
` : ""} ` : this.renderInactiveCrawlMessage()} ${when( isRunning, () => html`
${this.renderExclusions()}
(this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${this.isDialogVisible ? this.renderEditScale() : ""} ` )} `; }; private renderInactiveWatchCrawl() { return html`

${msg("Crawl is not currently running.")}

${when( this.workflow?.lastCrawlId, () => html` ${msg("Replay Latest Crawl")} ` )} this.runNow()}> ${msg("Run Crawl")}
`; } private renderInactiveCrawlMessage() { return html`

${msg("Crawl is not running.")}

`; } private renderExclusions() { return html`

${msg("Crawl URLs")}

(this.openDialogName = "exclusions")} > ${msg("Edit Exclusions")}
${when( this.lastCrawlId, () => html` ` )} (this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${this.workflow && this.isDialogVisible ? html`` : ""}
(this.openDialogName = undefined)} >${msg("Done Editing")}
`; } private renderEditScale() { if (!this.workflow) return; const scaleOptions = [ { value: 1, label: "1", }, { value: 2, label: "2", }, { value: 3, label: "3", }, ]; return html`
${scaleOptions.map( ({ value, label }) => html` { await this.scale(value); this.openDialogName = undefined; }} ?disabled=${this.isSubmittingUpdate} >${label} ` )}
(this.openDialogName = undefined)} >${msg("Cancel")}
`; } private renderSettings() { return html`
`; } private showDialog = async () => { await this.getWorkflowPromise; this.isDialogVisible = true; }; private handleExclusionChange(e: CustomEvent) { this.fetchWorkflow(); } private async scale(value: Crawl["scale"]) { if (!this.lastCrawlId) return; this.isSubmittingUpdate = true; try { const data = await this.apiFetch( `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/scale`, this.authState!, { method: "POST", body: JSON.stringify({ scale: +value }), } ); if (data.scaled) { this.fetchWorkflow(); this.notify({ message: msg("Updated crawl scale."), variant: "success", icon: "check2-circle", }); } else { throw new Error("unhandled API response"); } } catch { this.notify({ message: msg("Sorry, couldn't change crawl scale at this time."), variant: "danger", icon: "exclamation-octagon", }); } this.isSubmittingUpdate = false; } private async getWorkflow(): Promise { const data: Workflow = await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${this.workflowId}`, this.authState! ); return data; } private async fetchCrawls() { try { this.crawls = await this.getCrawls(); } catch { this.notify({ message: msg("Sorry, couldn't get crawls at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async getCrawls(): Promise { const query = queryString.stringify( { state: this.filterBy.state || inactiveCrawlStates, cid: this.workflowId, sortBy: "started", }, { arrayFormat: "comma", } ); const data: APIPaginatedList = await this.apiFetch( `/orgs/${this.orgId}/crawls?${query}`, this.authState! ); return data; } private async fetchCurrentCrawlStats() { if (!this.lastCrawlId) return; try { // TODO see if API can pass stats in GET workflow const { stats } = await this.getCrawl(this.lastCrawlId); this.lastCrawlStats = stats; } catch (e) { // TODO handle error console.debug(e); } } private stopPoll() { window.clearTimeout(this.timerId); } private async getCrawl(crawlId: Crawl["id"]): Promise { const data = await this.apiFetch( `/orgs/${this.orgId}/crawls/${crawlId}/replay.json`, this.authState! ); return data; } /** * Create a new template using existing template data */ private async duplicateConfig() { if (!this.workflow) return; const workflowParams: WorkflowParams = { ...this.workflow, name: msg(str`${this.renderName()} Copy`), }; this.navTo( `/orgs/${this.orgId}/workflows?new&jobType=${workflowParams.jobType}`, { workflow: workflowParams, } ); this.notify({ message: msg(str`Copied Workflow to new template.`), variant: "success", icon: "check2-circle", }); } private async deactivate(): Promise { if (!this.workflow) return; try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${this.workflow.id}`, this.authState!, { method: "DELETE", } ); this.workflow = { ...this.workflow, inactive: true, }; this.notify({ message: msg(html`Deactivated ${this.renderName()}.`), variant: "success", icon: "check2-circle", }); } catch { this.notify({ message: msg("Sorry, couldn't deactivate Workflow at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async delete(): Promise { if (!this.workflow) return; const isDeactivating = this.workflow.crawlCount > 0; try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${this.workflow.id}`, this.authState!, { method: "DELETE", } ); this.navTo(`/orgs/${this.orgId}/workflows/crawls`); this.notify({ message: isDeactivating ? msg(html`Deactivated ${this.renderName()}.`) : msg(html`Deleted ${this.renderName()}.`), variant: "success", icon: "check2-circle", }); } catch { this.notify({ message: isDeactivating ? msg("Sorry, couldn't deactivate Workflow at this time.") : msg("Sorry, couldn't delete Workflow at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async cancel() { if (!this.lastCrawlId) return; this.isCancelingOrStoppingCrawl = true; try { const data = await this.apiFetch( `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/cancel`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchWorkflow(); } else { throw data; } } catch { this.notify({ message: msg("Something went wrong, couldn't cancel crawl."), variant: "danger", icon: "exclamation-octagon", }); } this.isCancelingOrStoppingCrawl = false; } private async stop() { if (!this.lastCrawlId) return; this.isCancelingOrStoppingCrawl = true; try { const data = await this.apiFetch( `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/stop`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchWorkflow(); } else { throw data; } } catch { this.notify({ message: msg("Something went wrong, couldn't stop crawl."), variant: "danger", icon: "exclamation-octagon", }); } this.isCancelingOrStoppingCrawl = false; } private async runNow(): Promise { try { const data = await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${this.workflow!.id}/run`, this.authState!, { method: "POST", } ); this.lastCrawlId = data.started; // remove 'Z' from timestamp to match API response this.lastCrawlStartTime = new Date().toISOString().slice(0, -1); this.fetchWorkflow(); this.goToTab("watch"); this.notify({ message: msg("Starting crawl."), variant: "success", icon: "check2-circle", duration: 8000, }); } catch { this.notify({ message: msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async deleteCrawl(crawl: Crawl) { try { const data = await this.apiFetch( `/orgs/${crawl.oid}/crawls/delete`, this.authState!, { method: "POST", body: JSON.stringify({ crawl_ids: [crawl.id], }), } ); this.crawls = { ...this.crawls!, items: this.crawls!.items.filter((c) => c.id !== crawl.id), }; this.notify({ message: msg(`Successfully deleted crawl`), variant: "success", icon: "check2-circle", }); this.fetchCrawls(); } catch (e: any) { this.notify({ message: (e.isApiError && e.message) || msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } customElements.define("btrix-workflow-detail", WorkflowDetail);