import { state, property, customElement } from "lit/decorators.js"; import { msg, localized, str } from "@lit/localize"; import { when } from "lit/directives/when.js"; import { ifDefined } from "lit/directives/if-defined.js"; import queryString from "query-string"; import type { AuthState } from "@/utils/AuthService"; import LiteElement, { html } from "@/utils/LiteElement"; import type { ListWorkflow, Seed, Workflow, WorkflowParams } from "./types"; import { CopyButton } from "@/components/ui/copy-button"; import type { SlCheckbox } from "@shoelace-style/shoelace"; import type { APIPaginatedList, APIPaginationQuery } from "@/types/api"; import type { PageChangeEvent } from "@/components/ui/pagination"; import type { SelectNewDialogEvent } from "./index"; type SearchFields = "name" | "firstSeed"; type SortField = "lastRun" | "name" | "firstSeed" | "created" | "modified"; type SortDirection = "asc" | "desc"; const FILTER_BY_CURRENT_USER_STORAGE_KEY = "btrix.filterByCurrentUser.crawlConfigs"; const INITIAL_PAGE_SIZE = 10; const POLL_INTERVAL_SECONDS = 10; const ABORT_REASON_THROTTLE = "throttled"; // NOTE Backend pagination max is 1000 const SEEDS_MAX = 1000; const sortableFields: Record< SortField, { label: string; defaultDirection?: SortDirection } > = { lastRun: { label: msg("Latest Crawl"), defaultDirection: "desc", }, modified: { label: msg("Last Modified"), defaultDirection: "desc", }, name: { label: msg("Name"), defaultDirection: "asc", }, firstSeed: { label: msg("Crawl Start URL"), defaultDirection: "asc", }, created: { label: msg("Created"), defaultDirection: "desc", }, }; /** * Usage: * ```ts * * ``` */ @localized() @customElement("btrix-workflows-list") export class WorkflowsList extends LiteElement { static FieldLabels: Record = { name: msg("Name"), firstSeed: msg("Crawl Start URL"), }; @property({ type: Object }) authState!: AuthState; @property({ type: String }) orgId!: string; @property({ type: Boolean }) orgStorageQuotaReached = false; @property({ type: Boolean }) orgExecutionMinutesQuotaReached = false; @property({ type: String }) userId!: string; @property({ type: Boolean }) isCrawler!: boolean; @state() private workflows?: APIPaginatedList; @state() private searchOptions: any[] = []; @state() private isFetching = false; @state() private fetchErrorStatusCode?: number; @state() private orderBy: { field: SortField; direction: SortDirection; } = { field: "lastRun", direction: sortableFields["lastRun"].defaultDirection!, }; @state() private filterBy: Partial> = {}; @state() private filterByCurrentUser = false; // For fuzzy search: private searchKeys = ["name", "firstSeed"]; // Use to cancel requests private getWorkflowsController: AbortController | null = null; private timerId?: number; private get selectedSearchFilterKey() { return Object.keys(WorkflowsList.FieldLabels).find((key) => Boolean((this.filterBy as any)[key]) ); } constructor() { super(); this.filterByCurrentUser = window.sessionStorage.getItem(FILTER_BY_CURRENT_USER_STORAGE_KEY) === "true"; } protected async willUpdate(changedProperties: Map) { if (changedProperties.has("orgId")) { this.fetchConfigSearchValues(); } if ( changedProperties.has("orgId") || changedProperties.has("orderBy") || changedProperties.has("filterByCurrentUser") || changedProperties.has("filterByScheduled") || changedProperties.has("filterBy") ) { this.fetchWorkflows({ page: changedProperties.has("orgId") ? 1 : undefined, }); } if (changedProperties.has("filterByCurrentUser")) { window.sessionStorage.setItem( FILTER_BY_CURRENT_USER_STORAGE_KEY, this.filterByCurrentUser.toString() ); } } disconnectedCallback(): void { this.cancelInProgressGetWorkflows(); super.disconnectedCallback(); } private async fetchWorkflows(params?: APIPaginationQuery) { this.fetchErrorStatusCode = undefined; this.cancelInProgressGetWorkflows(); this.isFetching = true; try { const workflows = await this.getWorkflows(params); this.workflows = workflows; } catch (e: any) { if (e.isApiError) { this.fetchErrorStatusCode = e.statusCode; } else if (e.name === "AbortError") { console.debug("Fetch archived items aborted to throttle"); } else { this.notify({ message: msg("Sorry, couldn't retrieve Workflows at this time."), variant: "danger", icon: "exclamation-octagon", }); } } this.isFetching = false; // Restart timer for next poll this.timerId = window.setTimeout(() => { this.fetchWorkflows(); }, 1000 * POLL_INTERVAL_SECONDS); } private cancelInProgressGetWorkflows() { window.clearTimeout(this.timerId); if (this.getWorkflowsController) { this.getWorkflowsController.abort(ABORT_REASON_THROTTLE); this.getWorkflowsController = null; } } render() { return html`

${msg("Crawl Workflows")}

${when( this.isCrawler, () => html` { this.dispatchEvent( new CustomEvent("select-new-dialog", { detail: "workflow", }) ); }} > ${msg("New Workflow")} ` )}
${this.renderControls()}
${when( this.fetchErrorStatusCode, () => html`
${msg( `Something unexpected went wrong while retrieving Workflows.` )}
`, () => this.workflows ? this.workflows.total ? this.renderWorkflowList() : this.renderEmptyState() : this.renderLoading() )} `; } private renderControls() { return html`
${this.renderSearch()}
${msg("Sort by:")}
{ const field = (e.target as HTMLSelectElement).value as SortField; this.orderBy = { field: field, direction: sortableFields[field].defaultDirection || this.orderBy.direction, }; }} > ${Object.entries(sortableFields).map( ([value, { label }]) => html` ${label} ` )} { this.orderBy = { ...this.orderBy, direction: this.orderBy.direction === "asc" ? "desc" : "asc", }; }} >
`; } private renderSearch() { return html` { const { key, value } = e.detail; this.filterBy = { [key]: value, }; }} @btrix-clear=${() => { const { name: _name, firstSeed: _firstSeed, ...otherFilters } = this.filterBy; this.filterBy = otherFilters; }} > `; } private renderWorkflowList() { if (!this.workflows) return; const { page, total, pageSize } = this.workflows; return html` ${this.workflows.items.map(this.renderWorkflowItem)} ${when( total > pageSize, () => html`
{ await this.fetchWorkflows({ page: e.detail.page, }); // Scroll to top of list // TODO once deep-linking is implemented, scroll to top of pushstate this.scrollIntoView({ behavior: "smooth" }); }} >
` )} `; } private renderWorkflowItem = (workflow: ListWorkflow) => html` ${this.renderMenuItems(workflow)} `; private renderMenuItems(workflow: ListWorkflow) { return html` ${when( workflow.isCrawlRunning && this.isCrawler, // HACK shoelace doesn't current have a way to override non-hover // color without resetting the --sl-color-neutral-700 variable () => html` this.stop(workflow.lastCrawlId)} ?disabled=${workflow.lastCrawlStopping} > ${msg("Stop Crawl")} this.cancel(workflow.lastCrawlId)} > ${msg("Cancel & Discard Crawl")} ` )} ${when( this.isCrawler && !workflow.isCrawlRunning, () => html` this.runNow(workflow)} > ${msg("Run Crawl")} ` )} ${when( workflow.isCrawlRunning && this.isCrawler, // HACK shoelace doesn't current have a way to override non-hover // color without resetting the --sl-color-neutral-700 variable () => html` this.navTo( `${this.orgBasePath}/workflows/crawl/${workflow.id}#watch`, { dialog: "scale", } )} > ${msg("Edit Crawler Instances")} this.navTo( `${this.orgBasePath}/workflows/crawl/${workflow.id}#watch`, { dialog: "exclusions", } )} > ${msg("Edit Exclusions")} ` )} ${when( this.isCrawler, () => html` this.navTo( `${this.orgBasePath}/workflows/crawl/${workflow.id}?edit` )} > ${msg("Edit Workflow Settings")} ` )} CopyButton.copyToClipboard(workflow.tags.join(", "))} ?disabled=${!workflow.tags.length} > ${msg("Copy Tags")} ${when( this.isCrawler, () => html` this.duplicateConfig(workflow)} > ${msg("Duplicate Workflow")} ` )} `; } private renderName(crawlConfig: ListWorkflow) { if (crawlConfig.name) return crawlConfig.name; const { firstSeed, seedCount } = crawlConfig; if (seedCount === 1) { return firstSeed; } const remainderCount = seedCount - 1; if (remainderCount === 1) { return msg( html`${firstSeed} +${remainderCount} URL` ); } return msg( html`${firstSeed} +${remainderCount} URLs` ); } private renderEmptyState() { if (Object.keys(this.filterBy).length) { return html`

${msg("No matching Workflows found.")}

`; } if (this.workflows?.page && this.workflows?.page > 1) { return html`

${msg("Could not find page.")}

`; } if (this.isFetching) { return this.renderLoading(); } return html`

${msg("No Workflows yet.")}

`; } private renderLoading() { return html`
`; } /** * Fetch Workflows and update state **/ private async getWorkflows( queryParams?: APIPaginationQuery & Record ) { const query = queryString.stringify( { ...this.filterBy, page: queryParams?.page || this.workflows?.page || 1, pageSize: queryParams?.pageSize || this.workflows?.pageSize || INITIAL_PAGE_SIZE, userid: this.filterByCurrentUser ? this.userId : undefined, sortBy: this.orderBy.field, sortDirection: this.orderBy.direction === "desc" ? -1 : 1, }, { arrayFormat: "comma", } ); this.getWorkflowsController = new AbortController(); const data = await this.apiFetch>( `/orgs/${this.orgId}/crawlconfigs?${query}`, this.authState!, { signal: this.getWorkflowsController.signal, } ); this.getWorkflowsController = null; return data; } /** * Create a new template using existing template data */ private async duplicateConfig(workflow: ListWorkflow) { const [fullWorkflow, seeds] = await Promise.all([ this.getWorkflow(workflow), this.getSeeds(workflow), ]); const workflowParams: WorkflowParams = { ...fullWorkflow, name: workflow.name ? msg(str`${workflow.name} Copy`) : "", }; this.navTo( `${this.orgBasePath}/workflows?new&jobType=${workflowParams.jobType}`, { workflow: workflowParams, seeds: seeds.items, } ); if (seeds.total > SEEDS_MAX) { this.notify({ title: msg(str`Partially copied Workflow`), message: msg( str`Only first ${SEEDS_MAX.toLocaleString()} URLs were copied.` ), variant: "warning", icon: "exclamation-triangle", }); } else { this.notify({ message: msg(str`Copied Workflow to new template.`), variant: "success", icon: "check2-circle", }); } } private async deactivate(workflow: ListWorkflow): Promise { try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}`, this.authState!, { method: "DELETE", } ); this.fetchWorkflows(); this.notify({ message: msg( html`Deactivated ${this.renderName(workflow)}.` ), variant: "success", icon: "check2-circle", }); } catch { this.notify({ message: msg("Sorry, couldn't deactivate Workflow at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async delete(workflow: ListWorkflow): Promise { try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}`, this.authState!, { method: "DELETE", } ); this.fetchWorkflows(); this.notify({ message: msg( html`Deleted ${this.renderName(workflow)}.` ), variant: "success", icon: "check2-circle", }); } catch { this.notify({ message: msg("Sorry, couldn't delete Workflow at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async cancel(crawlId: ListWorkflow["lastCrawlId"]) { if (!crawlId) return; if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) { const data = await this.apiFetch<{ success: boolean }>( `/orgs/${this.orgId}/crawls/${crawlId}/cancel`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchWorkflows(); } else { this.notify({ message: msg("Something went wrong, couldn't cancel crawl."), variant: "danger", icon: "exclamation-octagon", }); } } } private async stop(crawlId: ListWorkflow["lastCrawlId"]) { if (!crawlId) return; if (window.confirm(msg("Are you sure you want to stop the crawl?"))) { const data = await this.apiFetch<{ success: boolean }>( `/orgs/${this.orgId}/crawls/${crawlId}/stop`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchWorkflows(); } else { this.notify({ message: msg("Something went wrong, couldn't stop crawl."), variant: "danger", icon: "exclamation-octagon", }); } } } private async runNow(workflow: ListWorkflow): Promise { try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}/run`, this.authState!, { method: "POST", } ); this.notify({ message: msg( html`Started crawl from ${this.renderName(workflow)}.
Watch crawl` ), variant: "success", icon: "check2-circle", duration: 8000, }); await this.fetchWorkflows(); // Scroll to top of list this.scrollIntoView({ behavior: "smooth" }); } catch (e: any) { let message = msg("Sorry, couldn't run crawl at this time."); if (e.isApiError && e.statusCode === 403) { if (e.details === "storage_quota_reached") { message = msg("Your org does not have enough storage to run crawls."); } else if (e.details === "exec_minutes_quota_reached") { message = msg( "Your org has used all of its execution minutes for this month." ); } else { message = msg("You do not have permission to run crawls."); } } this.notify({ message: message, variant: "danger", icon: "exclamation-octagon", }); } } private async fetchConfigSearchValues() { try { const data: { crawlIds: string[]; names: string[]; descriptions: string[]; firstSeeds: string[]; } = await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/search-values`, this.authState! ); // Update search/filter collection const toSearchItem = (key: SearchFields) => (value: string) => ({ [key]: value, }); this.searchOptions = [ ...data.names.map(toSearchItem("name")), ...data.firstSeeds.map(toSearchItem("firstSeed")), ]; } catch (e) { console.debug(e); } } private async getWorkflow(workflow: ListWorkflow): Promise { const data: Workflow = await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}`, this.authState! ); return data; } private async getSeeds(workflow: ListWorkflow) { // NOTE Returns first 1000 seeds (backend pagination max) const data = await this.apiFetch>( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}/seeds`, this.authState! ); return data; } }