import { localized, msg, str } from "@lit/localize"; import type { SlSelect } from "@shoelace-style/shoelace"; import { html, type PropertyValues, type TemplateResult } from "lit"; import { customElement, property, state } from "lit/decorators.js"; import { choose } from "lit/directives/choose.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { until } from "lit/directives/until.js"; import { when } from "lit/directives/when.js"; import queryString from "query-string"; import type { Crawl, Seed, Workflow, WorkflowParams } from "./types"; import { BtrixElement } from "@/classes/BtrixElement"; import type { PageChangeEvent } from "@/components/ui/pagination"; import { ClipboardController } from "@/controllers/clipboard"; import type { CrawlLog } from "@/features/archived-items/crawl-logs"; import { CrawlStatus } from "@/features/archived-items/crawl-status"; import { ExclusionEditor } from "@/features/crawl-workflows/exclusion-editor"; import { pageNav, type Breadcrumb } from "@/layouts/pageHeader"; import { deleteConfirmation } from "@/strings/ui"; import type { APIPaginatedList } from "@/types/api"; import { type CrawlState } from "@/types/crawlState"; import { isApiError } from "@/utils/api"; import { DEFAULT_MAX_SCALE, inactiveCrawlStates, isActive, } from "@/utils/crawler"; import { humanizeSchedule } from "@/utils/cron"; import { isArchivingDisabled } from "@/utils/orgs"; const SECTIONS = ["crawls", "watch", "settings", "logs"] as const; type Tab = (typeof SECTIONS)[number]; const DEFAULT_SECTION: Tab = "crawls"; const POLL_INTERVAL_SECONDS = 10; const LOGS_PAGE_SIZE = 50; /** * Usage: * ```ts * * ``` */ @customElement("btrix-workflow-detail") @localized() export class WorkflowDetail extends BtrixElement { @property({ type: String }) workflowId!: string; @property({ type: Boolean }) isEditing = false; @property({ type: Boolean }) isCrawler!: boolean; @property({ type: String }) openDialogName?: | "scale" | "exclusions" | "cancel" | "stop" | "delete" | "deleteCrawl"; @property({ type: String }) initialActivePanel?: Tab; @property({ type: Number }) maxScale = DEFAULT_MAX_SCALE; @state() private workflow?: Workflow; @state() private seeds?: APIPaginatedList; @state() private crawls?: APIPaginatedList; // Only inactive crawls @state() private logs?: APIPaginatedList; @state() private lastCrawlId: Workflow["lastCrawlId"] = null; @state() private lastCrawlStartTime: Workflow["lastCrawlStartTime"] = null; @state() private lastCrawlStats?: Crawl["stats"]; @state() private activePanel: Tab | undefined = SECTIONS[0]; @state() private isLoading = false; @state() private isSubmittingUpdate = false; @state() private isDialogVisible = false; @state() private isCancelingOrStoppingCrawl = false; @state() private crawlToDelete: Crawl | null = null; @state() private filterBy: Partial> = {}; private timerId?: number; private getWorkflowPromise?: Promise; private getSeedsPromise?: Promise>; private get isExplicitRunning() { return ( this.workflow?.isCrawlRunning && !this.workflow.lastCrawlStopping && this.workflow.lastCrawlState === "running" ); } private readonly tabLabels: Record = { crawls: msg("Crawls"), watch: msg("Watch Crawl"), logs: msg("Error Logs"), settings: msg("Settings"), }; connectedCallback(): void { // Set initial active section and dialog based on URL #hash value if (this.initialActivePanel) { this.activePanel = this.initialActivePanel; } else { void this.getActivePanelFromHash(); } super.connectedCallback(); window.addEventListener("hashchange", this.getActivePanelFromHash); } disconnectedCallback(): void { this.stopPoll(); super.disconnectedCallback(); window.removeEventListener("hashchange", this.getActivePanelFromHash); } firstUpdated() { if ( this.openDialogName && (this.openDialogName === "scale" || this.openDialogName === "exclusions") ) { void this.showDialog(); } } willUpdate(changedProperties: PropertyValues & Map) { if ( (changedProperties.has("workflowId") && this.workflowId) || (changedProperties.get("isEditing") === true && !this.isEditing) ) { void this.fetchWorkflow(); void this.fetchSeeds(); } if (changedProperties.has("isEditing")) { if (this.isEditing) { this.stopPoll(); } else { void this.getActivePanelFromHash(); } } if ( !this.isEditing && changedProperties.has("activePanel") && this.activePanel ) { if (this.activePanel === "crawls") { void this.fetchCrawls(); } } } private readonly getActivePanelFromHash = async () => { await this.updateComplete; if (this.isEditing) return; const hashValue = window.location.hash.slice(1); if (SECTIONS.includes(hashValue as (typeof SECTIONS)[number])) { this.activePanel = hashValue as Tab; } else { this.goToTab(DEFAULT_SECTION, { replace: true }); } }; private goToTab(tab: Tab, { replace = false } = {}) { const path = `${window.location.href.split("#")[0]}#${tab}`; if (replace) { window.history.replaceState(null, "", path); } else { window.history.pushState(null, "", path); } this.activePanel = tab; } private async fetchWorkflow() { this.stopPoll(); this.isLoading = true; try { const prevLastCrawlId = this.lastCrawlId; this.getWorkflowPromise = this.getWorkflow(); this.workflow = await this.getWorkflowPromise; this.lastCrawlId = this.workflow.lastCrawlId; this.lastCrawlStartTime = this.workflow.lastCrawlStartTime; if (this.lastCrawlId) { if (this.workflow.isCrawlRunning) { void this.fetchCurrentCrawlStats(); void this.fetchCrawlLogs(); } else if (this.lastCrawlId !== prevLastCrawlId) { this.logs = undefined; void this.fetchCrawlLogs(); } } // TODO: Check if storage quota has been exceeded here by running // crawl?? } catch (e) { this.notify.toast({ message: isApiError(e) && e.statusCode === 404 ? msg("Workflow not found.") : msg("Sorry, couldn't retrieve Workflow at this time."), variant: "danger", icon: "exclamation-octagon", id: "workflow-retrieve-error", }); } this.isLoading = false; if (!this.isEditing) { // Restart timer for next poll this.timerId = window.setTimeout(() => { void this.fetchWorkflow(); }, 1000 * POLL_INTERVAL_SECONDS); } } render() { if (this.isEditing && this.isCrawler) { return html`
${when(this.workflow, this.renderEditor)}
`; } return html`
${this.renderBreadcrumbs()}
${when( this.workflow?.inactive, () => html` ${msg("Inactive")} `, )}
${when( this.isCrawler && this.workflow && !this.workflow.inactive, this.renderActions, )}
${this.renderDetails()}
${when(this.workflow, this.renderTabList, this.renderLoading)}
(this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${msg( "Pages currently being crawled will be completed and saved, and finished pages will be kept, but all remaining pages in the queue will be discarded. Are you sure you want to stop crawling?", )}
(this.openDialogName = undefined)} >${msg("Keep Crawling")} { await this.stop(); this.openDialogName = undefined; }} >${msg("Stop Crawling")}
(this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${msg( "Canceling will discard all pages crawled. Are you sure you want to discard them?", )}
(this.openDialogName = undefined)} >${msg("Keep Crawling")} { await this.cancel(); this.openDialogName = undefined; }} >${msg(html`Cancel & Discard Crawl`)}
(this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${msg( "All files and logs associated with this crawl will also be deleted, and the crawl will be removed from any Collection it is a part of.", )}
(this.openDialogName = undefined)} >${msg("Cancel")} { this.openDialogName = undefined; if (this.crawlToDelete) { await this.deleteCrawl(this.crawlToDelete); } }} >${msg("Delete Crawl")}
(this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${this.isDialogVisible ? this.renderEditScale() : ""} (this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${deleteConfirmation(this.renderName())}
(this.openDialogName = undefined)} >${msg("Cancel")} { void this.delete(); this.openDialogName = undefined; }} >${msg("Delete Workflow")}
`; } private renderBreadcrumbs() { const breadcrumbs: Breadcrumb[] = [ { href: `${this.navigate.orgBasePath}/workflows`, content: msg("Crawl Workflows"), }, ]; if (this.isEditing) { breadcrumbs.push( { href: `${this.navigate.orgBasePath}/workflows/${this.workflowId}`, content: this.workflow ? this.renderName() : undefined, }, { content: msg("Edit Settings"), }, ); } else { breadcrumbs.push({ content: this.workflow ? this.renderName() : undefined, }); } return pageNav(breadcrumbs); } private readonly renderTabList = () => html`
${this.renderPanelHeader()}
${this.renderTab("crawls")} ${this.renderTab("watch")} ${this.renderTab("logs")} ${this.renderTab("settings")} ${this.renderCrawls()} ${until( this.getWorkflowPromise?.then( () => html` ${when(this.activePanel === "watch", () => this.workflow?.isCrawlRunning ? html`
${this.renderCurrentCrawl()}
${this.renderWatchCrawl()}` : this.renderInactiveWatchCrawl(), )} `, ), )}
${this.renderLogs()} ${this.renderSettings()}
`; private renderPanelHeader() { if (!this.activePanel) return; if (this.activePanel === "crawls") { return html`

${this.tabLabels[this.activePanel]} ${when( this.crawls, () => html` (${this.localize.number(this.crawls!.total)}${this.workflow ?.isCrawlRunning ? html` + 1` : ""}) `, )}

`; } if (this.activePanel === "settings" && this.isCrawler) { return html`

${this.tabLabels[this.activePanel]}

this.navigate.to( `/orgs/${this.appState.orgSlug}/workflows/${this.workflow?.id}?edit`, )} > `; } if (this.activePanel === "watch" && this.isCrawler) { const enableEditBrowserWindows = this.workflow?.isCrawlRunning && !this.workflow.lastCrawlStopping; return html`

${this.tabLabels[this.activePanel]}

(this.openDialogName = "scale")} > ${msg("Edit Browser Windows")}
`; } if (this.activePanel === "logs") { const authToken = this.authState?.headers.Authorization.split(" ")[1]; const isDownloadEnabled = Boolean( this.logs?.total && this.workflow?.lastCrawlId && !this.workflow.isCrawlRunning, ); return html`

${this.tabLabels[this.activePanel]}

${msg("Download Logs")} `; } return html`

${this.tabLabels[this.activePanel]}

`; } private renderTab(tabName: Tab, { disabled = false } = {}) { const isActive = tabName === this.activePanel; return html` { if (disabled) e.preventDefault(); }} > ${choose(tabName, [ [ "crawls", () => html``, ], ["watch", () => html``], ["logs", () => html``], ["settings", () => html``], ])} ${this.tabLabels[tabName]} `; } private readonly renderEditor = () => html`
${this.renderBreadcrumbs()}
${when( !this.isLoading && this.seeds && this.workflow, (workflow) => html` this.navigate.to( `${this.navigate.orgBasePath}/workflows/${workflow.id}`, )} > `, this.renderLoading, )} `; private readonly renderActions = () => { if (!this.workflow) return; const workflow = this.workflow; const archivingDisabled = isArchivingDisabled(this.org, true); return html` ${when( this.workflow.isCrawlRunning, () => html` (this.openDialogName = "stop")} ?disabled=${!this.lastCrawlId || this.isCancelingOrStoppingCrawl || this.workflow?.lastCrawlStopping} > ${msg("Stop")} (this.openDialogName = "cancel")} ?disabled=${!this.lastCrawlId || this.isCancelingOrStoppingCrawl} > ${msg("Cancel")} `, () => html` void this.runNow()} > ${msg("Run Crawl")} `, )} ${msg("Actions")} ${when( this.workflow.isCrawlRunning, // HACK shoelace doesn't current have a way to override non-hover // color without resetting the --sl-color-neutral-700 variable () => html` (this.openDialogName = "stop")} ?disabled=${workflow.lastCrawlStopping || this.isCancelingOrStoppingCrawl} > ${msg("Stop Crawl")} (this.openDialogName = "cancel")} > ${msg(html`Cancel & Discard Crawl`)} `, () => html` void this.runNow()} > ${msg("Run Crawl")} `, )} ${when( workflow.isCrawlRunning && !workflow.lastCrawlStopping, () => html` (this.openDialogName = "scale")}> ${msg("Edit Browser Windows")} (this.openDialogName = "exclusions")} ?disabled=${!this.isExplicitRunning} > ${msg("Edit Exclusions")} `, )} this.navigate.to( `/orgs/${this.appState.orgSlug}/workflows/${workflow.id}?edit`, )} > ${msg("Edit Workflow Settings")} ClipboardController.copyToClipboard(workflow.tags.join(", "))} ?disabled=${!workflow.tags.length} > ${msg("Copy Tags")} void this.duplicateConfig()} > ${msg("Duplicate Workflow")} ${when( !workflow.crawlCount, () => html` (this.openDialogName = "delete")} > ${msg("Delete Workflow")} `, )} `; }; private renderDetails() { return html` ${this.renderDetailItem( msg("Status"), (workflow) => html` `, )} ${this.renderDetailItem( msg("Total Size"), (workflow) => html` ${this.localize.bytes(Number(workflow.totalSize), { unitDisplay: "narrow", })}`, )} ${this.renderDetailItem(msg("Schedule"), (workflow) => workflow.schedule ? html`
${humanizeSchedule(workflow.schedule, { length: "short", })}
` : html`${msg("No Schedule")}`, )} ${this.renderDetailItem(msg("Created By"), (workflow) => msg( str`${workflow.createdByName} on ${this.localize.date( new Date(workflow.created), { year: "numeric", month: "numeric", day: "numeric", }, )}`, ), )}
`; } private renderDetailItem( label: string | TemplateResult, renderContent: (workflow: Workflow) => TemplateResult | string | number, ) { return html` ${when( this.workflow, renderContent, () => html``, )} `; } private renderName() { if (!this.workflow) return html``; if (this.workflow.name) return html`${this.workflow.name}`; const { seedCount, firstSeed } = this.workflow; if (seedCount === 1) { return html`${firstSeed}`; } const remainderCount = seedCount - 1; if (remainderCount === 1) { return msg( html` ${firstSeed} +${remainderCount} URL`, ); } return msg( html` ${firstSeed} +${remainderCount} URLs`, ); } private renderCrawls() { return html`
${msg("View:")}
{ const value = (e.target as SlSelect).value as CrawlState[]; await this.updateComplete; this.filterBy = { ...this.filterBy, state: value, }; void this.fetchCrawls(); }} > ${inactiveCrawlStates.map(this.renderStatusMenuItem)}
${when( this.workflow?.isCrawlRunning, () => html`
${msg( html`Crawl is currently running. Watch Crawl Progress`, )}
`, )}
${when(this.crawls, () => this.crawls!.items.map( (crawl: Crawl) => html` ${when( this.isCrawler, () => html` this.confirmDeleteCrawl(crawl)} > ${msg("Delete Crawl")} `, )}`, ), )}
${when( this.crawls && !this.crawls.items.length, () => html`

${this.crawls?.total ? msg("No matching crawls found.") : msg("No crawls yet.")}

`, )}
`; } private readonly renderStatusMenuItem = (state: CrawlState) => { const { icon, label } = CrawlStatus.getContent(state); return html`${icon}${label}`; }; private readonly renderCurrentCrawl = () => { const skeleton = html``; return html` ${this.renderDetailItem(msg("Pages Crawled"), () => this.lastCrawlStats ? `${this.localize.number( +(this.lastCrawlStats.done || 0), )} / ${this.localize.number(+(this.lastCrawlStats.found || 0))}` : html``, )} ${this.renderDetailItem(msg("Run Duration"), () => this.lastCrawlStartTime ? this.localize.humanizeDuration( new Date().valueOf() - new Date(this.lastCrawlStartTime).valueOf(), ) : skeleton, )} ${this.renderDetailItem(msg("Crawl Size"), () => this.workflow ? this.localize.bytes(this.workflow.lastCrawlSize || 0, { unitDisplay: "narrow", }) : skeleton, )} ${this.renderDetailItem(msg("Browser Windows"), () => this.workflow && this.appState.settings ? this.workflow.scale * this.appState.settings.numBrowsers : skeleton, )} `; }; private readonly renderWatchCrawl = () => { if (!this.authState || !this.workflow?.lastCrawlState) return ""; // Show custom message if crawl is active but not explicitly running let waitingMsg: string | null = null; if (!this.isExplicitRunning) { switch (this.workflow.lastCrawlState) { case "starting": waitingMsg = msg("Crawl starting..."); break; case "waiting_capacity": waitingMsg = msg( "Crawl waiting for available resources before it can continue...", ); break; case "waiting_org_limit": waitingMsg = msg( "Crawl waiting for others to finish, concurrent limit per Organization reached...", ); break; case "pending-wait": case "generate-wacz": case "uploading-wacz": waitingMsg = msg("Crawl finishing..."); break; default: if (this.workflow.lastCrawlStopping) { waitingMsg = msg("Crawl stopping..."); } break; } } const authToken = this.authState.headers.Authorization.split(" ")[1]; return html` ${when( this.isExplicitRunning && this.workflow, (workflow) => html`
${this.renderCrawlErrors()}
${this.renderExclusions()}
`, () => waitingMsg ? html`

${waitingMsg}

` : this.renderInactiveCrawlMessage(), )} `; }; private renderInactiveWatchCrawl() { return html`

${msg("Crawl workflow is not currently running.")}

${when( this.workflow?.lastCrawlId && this.workflow, (workflow) => html` ${msg("Replay Latest Crawl")} `, )} ${when( this.isCrawler && this.workflow, (workflow) => html` ${msg("QA Latest Crawl")} `, )}
`; } private renderInactiveCrawlMessage() { return html`

${msg("Crawl is not running.")}

`; } private renderLogs() { return html`
${when( this.workflow?.isCrawlRunning, () => html`
${msg( html`Viewing error logs for currently running crawl. Watch Crawl Progress`, )}
`, )} ${when( this.lastCrawlId, () => this.logs?.total ? html` { await this.fetchCrawlLogs({ page: e.detail.page, }); // Scroll to top of list this.scrollIntoView(); }} >` : html`

${this.workflow?.lastCrawlState === "waiting_capacity" ? msg("Error logs currently not available.") : msg("No error logs found yet for latest crawl.")}

`, () => this.renderNoCrawlLogs(), )}
`; } private renderNoCrawlLogs() { return html`

${msg("Logs will show here after you run a crawl.")}

void this.runNow()} > ${msg("Run Crawl")}
`; } private renderCrawlErrors() { return html`

${msg("Error Logs")} ${this.logs?.total ? this.localize.number(this.logs.total) : 0}

${when( this.logs?.total && this.logs.total > LOGS_PAGE_SIZE, () => html`

${msg( str`Displaying latest ${this.localize.number(LOGS_PAGE_SIZE)} errors of ${this.localize.number(this.logs!.total)}.`, )}

`, )}
`; } private renderExclusions() { return html`

${msg("Upcoming Pages")}

(this.openDialogName = "exclusions")} > ${msg("Edit Exclusions")}
${when( this.lastCrawlId, () => html` `, )} (this.openDialogName = undefined)} @sl-show=${this.showDialog} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${this.workflow && this.isDialogVisible ? html`` : ""}
${msg("Done Editing")}
`; } private renderEditScale() { if (!this.workflow) return; const scaleOptions = []; if (this.appState.settings) { for (let value = 1; value <= this.maxScale; value++) { scaleOptions.push({ value, label: value * this.appState.settings.numBrowsers, }); } } return html`

${msg( "Change the number of browser windows crawling in parallel. This change will take effect immediately on the currently running crawl and update crawl workflow settings.", )}

${scaleOptions.map( ({ value, label }) => html` { await this.scale(value); this.openDialogName = undefined; }} ?disabled=${this.isSubmittingUpdate} >${label} `, )}
(this.openDialogName = undefined)} >${msg("Cancel")}
`; } private renderSettings() { return html`
`; } private readonly renderLoading = () => html`
`; private readonly showDialog = async () => { await this.getWorkflowPromise; this.isDialogVisible = true; }; private handleExclusionChange() { void this.fetchWorkflow(); } private async scale(value: Crawl["scale"]) { if (!this.lastCrawlId) return; this.isSubmittingUpdate = true; try { const data = await this.api.fetch<{ scaled: boolean }>( `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/scale`, { method: "POST", body: JSON.stringify({ scale: +value }), }, ); if (data.scaled) { void this.fetchWorkflow(); this.notify.toast({ message: msg("Updated number of browser windows."), variant: "success", icon: "check2-circle", id: "browser-windows-update-status", }); } else { throw new Error("unhandled API response"); } } catch { this.notify.toast({ message: msg( "Sorry, couldn't change number of browser windows at this time.", ), variant: "danger", icon: "exclamation-octagon", id: "browser-windows-update-status", }); } this.isSubmittingUpdate = false; } private async getWorkflow(): Promise { const data: Workflow = await this.api.fetch( `/orgs/${this.orgId}/crawlconfigs/${this.workflowId}`, ); return data; } private async onCloseExclusions() { const editor = this.querySelector("btrix-exclusion-editor"); if (editor && editor instanceof ExclusionEditor) { await editor.onClose(); } this.openDialogName = undefined; } private async fetchSeeds(): Promise { try { this.getSeedsPromise = this.getSeeds(); this.seeds = await this.getSeedsPromise; } catch { this.notify.toast({ message: msg( "Sorry, couldn't retrieve all crawl settings at this time.", ), variant: "danger", icon: "exclamation-octagon", id: "archived-item-retrieve-error", }); } } private async getSeeds() { const data = await this.api.fetch>( `/orgs/${this.orgId}/crawlconfigs/${this.workflowId}/seeds`, ); return data; } private async fetchCrawls() { try { this.crawls = await this.getCrawls(); } catch { this.notify.toast({ message: msg("Sorry, couldn't get crawls at this time."), variant: "danger", icon: "exclamation-octagon", id: "archived-item-retrieve-error", }); } } private async getCrawls() { const query = queryString.stringify( { state: this.filterBy.state, cid: this.workflowId, sortBy: "started", }, { arrayFormat: "comma", }, ); const data = await this.api.fetch>( `/orgs/${this.orgId}/crawls?${query}`, ); return data; } private async fetchCurrentCrawlStats() { if (!this.lastCrawlId) return; try { // TODO see if API can pass stats in GET workflow const { stats } = await this.getCrawl(this.lastCrawlId); this.lastCrawlStats = stats; } catch (e) { // TODO handle error console.debug(e); } } private stopPoll() { window.clearTimeout(this.timerId); } private async getCrawl(crawlId: Crawl["id"]): Promise { const data = await this.api.fetch( `/orgs/${this.orgId}/crawls/${crawlId}/replay.json`, ); return data; } /** * Create a new template using existing template data */ private async duplicateConfig() { if (!this.workflow) await this.getWorkflowPromise; if (!this.seeds) await this.getSeedsPromise; await this.updateComplete; if (!this.workflow) return; const workflowParams: WorkflowParams = { ...this.workflow, name: this.workflow.name ? msg(str`${this.workflow.name} Copy`) : "", }; this.navigate.to(`${this.navigate.orgBasePath}/workflows/new`, { workflow: workflowParams, seeds: this.seeds?.items, }); this.notify.toast({ message: msg(str`Copied Workflow to new template.`), variant: "success", icon: "check2-circle", id: "workflow-copied-success", }); } private async delete(): Promise { if (!this.workflow) return; try { await this.api.fetch( `/orgs/${this.orgId}/crawlconfigs/${this.workflow.id}`, { method: "DELETE", }, ); this.navigate.to(`${this.navigate.orgBasePath}/workflows`); this.notify.toast({ message: msg( html`Deleted ${this.renderName()} Workflow.`, ), variant: "success", icon: "check2-circle", id: "workflow-delete-status", }); } catch { this.notify.toast({ message: msg("Sorry, couldn't delete Workflow at this time."), variant: "danger", icon: "exclamation-octagon", id: "workflow-delete-status", }); } } private async cancel() { if (!this.lastCrawlId) return; this.isCancelingOrStoppingCrawl = true; try { const data = await this.api.fetch<{ success: boolean }>( `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/cancel`, { method: "POST", }, ); if (data.success) { void this.fetchWorkflow(); } else { throw data; } } catch { this.notify.toast({ message: msg("Something went wrong, couldn't cancel crawl."), variant: "danger", icon: "exclamation-octagon", id: "crawl-stop-error", }); } this.isCancelingOrStoppingCrawl = false; } private async stop() { if (!this.lastCrawlId) return; this.isCancelingOrStoppingCrawl = true; try { const data = await this.api.fetch<{ success: boolean }>( `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/stop`, { method: "POST", }, ); if (data.success) { void this.fetchWorkflow(); } else { throw data; } } catch { this.notify.toast({ message: msg("Something went wrong, couldn't stop crawl."), variant: "danger", icon: "exclamation-octagon", id: "crawl-stop-error", }); } this.isCancelingOrStoppingCrawl = false; } private async runNow(): Promise { try { const data = await this.api.fetch<{ started: string | null }>( `/orgs/${this.orgId}/crawlconfigs/${this.workflowId}/run`, { method: "POST", }, ); this.lastCrawlId = data.started; this.lastCrawlStartTime = new Date().toISOString(); this.logs = undefined; void this.fetchWorkflow(); this.goToTab("watch"); this.notify.toast({ message: msg("Starting crawl."), variant: "success", icon: "check2-circle", id: "crawl-start-status", }); } catch (e) { let message = msg("Sorry, couldn't run crawl at this time."); if (isApiError(e) && e.statusCode === 403) { if (e.details === "storage_quota_reached") { message = msg("Your org does not have enough storage to run crawls."); } else if (e.details === "exec_minutes_quota_reached") { message = msg( "Your org has used all of its execution minutes for this month.", ); } else { message = msg("You do not have permission to run crawls."); } } else if (isApiError(e) && e.details == "proxy_not_found") { message = msg( "Your org doesn't have permission to use the proxy configured for this crawl.", ); } this.notify.toast({ message: message, variant: "danger", icon: "exclamation-octagon", id: "crawl-start-status", }); } } private readonly confirmDeleteCrawl = (crawl: Crawl) => { this.crawlToDelete = crawl; this.openDialogName = "deleteCrawl"; }; private async deleteCrawl(crawl: Crawl) { try { const _data = await this.api.fetch(`/orgs/${crawl.oid}/crawls/delete`, { method: "POST", body: JSON.stringify({ crawl_ids: [crawl.id], }), }); this.crawlToDelete = null; this.crawls = { ...this.crawls!, items: this.crawls!.items.filter((c) => c.id !== crawl.id), }; this.notify.toast({ message: msg(`Successfully deleted crawl`), variant: "success", icon: "check2-circle", id: "archived-item-delete-status", }); void this.fetchCrawls(); // Update crawl count void this.fetchWorkflow(); } catch (e) { if (this.crawlToDelete) { this.confirmDeleteCrawl(this.crawlToDelete); } let message = msg( str`Sorry, couldn't delete archived item at this time.`, ); if (isApiError(e)) { if (e.details == "not_allowed") { message = msg( str`Only org owners can delete other users' archived items.`, ); } else if (e.message) { message = e.message; } } this.notify.toast({ message: message, variant: "danger", icon: "exclamation-octagon", id: "archived-item-delete-status", }); } } private async fetchCrawlLogs( params: Partial = {}, ): Promise { try { this.logs = await this.getCrawlErrors(params); } catch (e) { if (isApiError(e) && e.statusCode === 503) { // do nothing, keep logs if previously loaded } else { this.notify.toast({ message: msg( "Sorry, couldn't retrieve crawl error logs at this time.", ), variant: "danger", icon: "exclamation-octagon", id: "archived-item-retrieve-error", }); } } } private async getCrawlErrors(params: Partial) { const page = params.page || this.logs?.page || 1; const pageSize = params.pageSize || this.logs?.pageSize || LOGS_PAGE_SIZE; const data = await this.api.fetch>( `/orgs/${this.orgId}/crawls/${ this.workflow!.lastCrawlId }/errors?page=${page}&pageSize=${pageSize}`, ); return data; } }