import type { HTMLTemplateResult, PropertyValueMap } from "lit"; import { state, property, query } from "lit/decorators.js"; import { msg, localized, str } from "@lit/localize"; import { when } from "lit/directives/when.js"; import debounce from "lodash/fp/debounce"; import flow from "lodash/fp/flow"; import map from "lodash/fp/map"; import orderBy from "lodash/fp/orderBy"; import filter from "lodash/fp/filter"; import Fuse from "fuse.js"; import queryString from "query-string"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import type { Crawl, Workflow, WorkflowParams } from "./types"; import { CopyButton } from "../../components/copy-button"; import { SlCheckbox } from "@shoelace-style/shoelace"; import type { APIPaginatedList } from "../../types/api"; type SortField = "_lastUpdated" | "_name"; type SortDirection = "asc" | "desc"; const FILTER_BY_CURRENT_USER_STORAGE_KEY = "btrix.filterByCurrentUser.crawlConfigs"; const INITIAL_PAGE_SIZE = 30; const POLL_INTERVAL_SECONDS = 10; const MIN_SEARCH_LENGTH = 2; const sortableFields: Record< SortField, { label: string; defaultDirection?: SortDirection } > = { _lastUpdated: { label: msg("Last Updated"), defaultDirection: "desc", }, _name: { label: msg("Name"), defaultDirection: "asc", }, }; /** * Usage: * ```ts * * ``` */ @localized() export class WorkflowsList extends LiteElement { @property({ type: Object }) authState!: AuthState; @property({ type: String }) orgId!: string; @property({ type: String }) userId!: string; @property({ type: Boolean }) isCrawler!: boolean; @state() private workflows?: Workflow[]; @state() private fetchErrorStatusCode?: number; @state() private orderBy: { field: SortField; direction: SortDirection; } = { field: "_lastUpdated", direction: sortableFields["_lastUpdated"].defaultDirection!, }; @state() private filterByCurrentUser = false; @state() private searchBy: string = ""; @state() private filterByScheduled: boolean | null = null; // For fuzzy search: private fuse = new Fuse([], { keys: ["name", "config.seeds", "config.seeds.url"], shouldSort: false, threshold: 0.2, // stricter; default is 0.6 }); private timerId?: number; constructor() { super(); this.filterByCurrentUser = window.sessionStorage.getItem(FILTER_BY_CURRENT_USER_STORAGE_KEY) === "true"; } protected async willUpdate(changedProperties: Map) { if ( changedProperties.has("orgId") || changedProperties.has("filterByCurrentUser") ) { this.fetchWorkflows(); } if (changedProperties.has("filterByCurrentUser")) { window.sessionStorage.setItem( FILTER_BY_CURRENT_USER_STORAGE_KEY, this.filterByCurrentUser.toString() ); } } disconnectedCallback(): void { this.cancelInProgressGetWorkflows(); super.disconnectedCallback(); } private async fetchWorkflows() { this.fetchErrorStatusCode = undefined; this.cancelInProgressGetWorkflows(); try { const workflows = await this.getWorkflows(); this.workflows = workflows; // Update search/filter collection this.fuse.setCollection(this.workflows as any); } catch (e: any) { if (e.isApiError) { this.fetchErrorStatusCode = e.statusCode; } else { this.notify({ message: msg("Sorry, couldn't retrieve Workflows at this time."), variant: "danger", icon: "exclamation-octagon", }); } } // Restart timer for next poll this.timerId = window.setTimeout(() => { this.fetchWorkflows(); }, 1000 * POLL_INTERVAL_SECONDS); } private cancelInProgressGetWorkflows() { window.clearTimeout(this.timerId); } render() { return html`

${msg("Crawling")}

${when( this.isCrawler, () => html` ${msg("New Crawl Workflow")} ` )}
${this.renderControls()}
${when( this.fetchErrorStatusCode, () => html`
${msg( `Something unexpected went wrong while retrieving Workflows.` )}
`, () => this.workflows ? this.workflows.length ? this.renderWorkflowList() : html`

${msg("No Workflows yet.")}

` : html`
` )} `; } private renderControls() { return html`
${msg("Sort by:")}
{ const field = (e.target as HTMLSelectElement).value as SortField; this.orderBy = { field: field, direction: sortableFields[field].defaultDirection || this.orderBy.direction, }; }} > ${Object.entries(sortableFields).map( ([value, { label }]) => html` ${label} ` )} { this.orderBy = { ...this.orderBy, direction: this.orderBy.direction === "asc" ? "desc" : "asc", }; }} >
`; } private renderWorkflowList() { if (!this.workflows) return; const flowFns = [ map((workflow: Workflow) => ({ ...workflow, _lastUpdated: this.workflowLastUpdated(workflow), _name: workflow.name || workflow.firstSeed, })), orderBy(this.orderBy.field, this.orderBy.direction), map(this.renderWorkflowItem), ]; if (this.filterByScheduled === true) { flowFns.unshift(filter(({ schedule }: any) => Boolean(schedule))); } else if (this.filterByScheduled === false) { flowFns.unshift(filter(({ schedule }: any) => !schedule)); } if (this.searchBy.length >= MIN_SEARCH_LENGTH) { flowFns.unshift(this.filterResults); } return html` ${flow(...flowFns)(this.workflows)} `; } private renderWorkflowItem = (workflow: Workflow) => html` ${this.renderMenuItems(workflow)} `; private renderMenuItems(workflow: Workflow) { return html` ${when( workflow.currCrawlId, // HACK shoelace doesn't current have a way to override non-hover // color without resetting the --sl-color-neutral-700 variable () => html` this.stop(workflow.currCrawlId)} ?disabled=${workflow.currCrawlStopping} > ${msg("Stop Crawl")} this.cancel(workflow.currCrawlId)} > ${msg("Cancel & Discard Crawl")} `, () => html` this.runNow(workflow)} > ${msg("Run Crawl")} ` )} ${when( workflow.currCrawlState === "running", () => html` this.navTo( `/orgs/${workflow.oid}/workflows/crawl/${workflow.id}#watch`, { dialog: "scale", } )} > ${msg("Edit Crawler Instances")} this.navTo( `/orgs/${workflow.oid}/workflows/crawl/${workflow.id}#watch`, { dialog: "exclusions", } )} > ${msg("Edit Exclusions")} ` )} this.navTo( `/orgs/${workflow.oid}/workflows/crawl/${workflow.id}?edit` )} > ${msg("Edit Workflow Settings")} CopyButton.copyToClipboard(workflow.tags.join(","))} ?disabled=${!workflow.tags.length} > ${msg("Copy Tags")} this.duplicateConfig(workflow)}> ${msg("Duplicate Workflow")} ${when(!workflow.currCrawlId, () => { const shouldDeactivate = workflow.crawlCount && !workflow.inactive; return html` shouldDeactivate ? this.deactivate(workflow) : this.delete(workflow)} > ${shouldDeactivate ? msg("Deactivate Workflow") : msg("Delete Workflow")} `; })} `; } private renderName(crawlConfig: Workflow) { if (crawlConfig.name) return crawlConfig.name; const { config } = crawlConfig; const firstSeed = config.seeds[0]; let firstSeedURL = firstSeed.url; if (config.seeds.length === 1) { return firstSeedURL; } const remainderCount = config.seeds.length - 1; if (remainderCount === 1) { return msg( html`${firstSeedURL} +${remainderCount} URL` ); } return msg( html`${firstSeedURL} +${remainderCount} URLs` ); } private workflowLastUpdated(workflow: Workflow): Date { return new Date( Math.max( ...[ workflow.currCrawlStartTime, workflow.lastCrawlTime, workflow.lastCrawlStartTime, workflow.modified, workflow.created, ] .filter((date) => date) .map((date) => new Date(`${date}Z`).getTime()) ) ); } private onSearchInput = debounce(200)((e: any) => { this.searchBy = e.target.value; }) as any; private filterResults = () => { const results = this.fuse.search(this.searchBy); return results.map(({ item }) => item); }; /** * Fetch Workflows and update state **/ private async getWorkflows(): Promise { const params = this.filterByCurrentUser ? `?userid=${this.userId}` : ""; const data: APIPaginatedList = await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs${params}`, this.authState! ); return data.items; } /** * Create a new template using existing template data */ private async duplicateConfig(workflow: Workflow) { const workflowParams: WorkflowParams = { ...workflow, name: msg(str`${this.renderName(workflow)} Copy`), }; this.navTo( `/orgs/${this.orgId}/workflows?new&jobType=${workflowParams.jobType}`, { workflow: workflowParams, } ); this.notify({ message: msg(str`Copied Workflow to new template.`), variant: "success", icon: "check2-circle", }); } private async deactivate(workflow: Workflow): Promise { try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}`, this.authState!, { method: "DELETE", } ); this.notify({ message: msg( html`Deactivated ${this.renderName(workflow)}.` ), variant: "success", icon: "check2-circle", }); this.workflows = this.workflows!.filter((t) => t.id !== workflow.id); } catch { this.notify({ message: msg("Sorry, couldn't deactivate Workflow at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async delete(workflow: Workflow): Promise { try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}`, this.authState!, { method: "DELETE", } ); this.notify({ message: msg( html`Deleted ${this.renderName(workflow)}.` ), variant: "success", icon: "check2-circle", }); this.workflows = this.workflows!.filter((t) => t.id !== workflow.id); } catch { this.notify({ message: msg("Sorry, couldn't delete Workflow at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async cancel(crawlId: Workflow["currCrawlId"]) { if (!crawlId) return; if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) { const data = await this.apiFetch( `/orgs/${this.orgId}/crawls/${crawlId}/cancel`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchWorkflows(); } else { this.notify({ message: msg("Something went wrong, couldn't cancel crawl."), variant: "danger", icon: "exclamation-octagon", }); } } } private async stop(crawlId: Workflow["currCrawlId"]) { if (!crawlId) return; if (window.confirm(msg("Are you sure you want to stop the crawl?"))) { const data = await this.apiFetch( `/orgs/${this.orgId}/crawls/${crawlId}/stop`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchWorkflows(); } else { this.notify({ message: msg("Something went wrong, couldn't stop crawl."), variant: "danger", icon: "exclamation-octagon", }); } } } private async runNow(workflow: Workflow): Promise { try { const data = await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}/run`, this.authState!, { method: "POST", } ); this.notify({ message: msg( html`Started crawl from ${this.renderName(workflow)}.
Watch crawl` ), variant: "success", icon: "check2-circle", duration: 8000, }); await this.fetchWorkflows(); // Scroll to top of list this.scrollIntoView({ behavior: "smooth" }); } catch (e: any) { this.notify({ message: (e.isApiError && e.statusCode === 403 && msg("You do not have permission to run crawls.")) || msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } customElements.define("btrix-workflows-list", WorkflowsList);