import type { HTMLTemplateResult, PropertyValueMap } from "lit"; import { state, property } from "lit/decorators.js"; import { msg, localized, str } from "@lit/localize"; import { when } from "lit/directives/when.js"; import debounce from "lodash/fp/debounce"; import flow from "lodash/fp/flow"; import map from "lodash/fp/map"; import orderBy from "lodash/fp/orderBy"; import filter from "lodash/fp/filter"; import Fuse from "fuse.js"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import type { Workflow, WorkflowParams } from "./types"; import { getUTCSchedule, humanizeNextDate, humanizeSchedule, } from "../../utils/cron"; import "../../components/crawl-scheduler"; import { SlCheckbox } from "@shoelace-style/shoelace"; import type { APIPaginatedList } from "../../types/api"; type RunningCrawlsMap = { /** Map of configId: crawlId */ [configId: string]: string; }; const FILTER_BY_CURRENT_USER_STORAGE_KEY = "btrix.filterByCurrentUser.crawlConfigs"; const MIN_SEARCH_LENGTH = 2; const sortableFieldLabels = { created_desc: msg("Newest"), created_asc: msg("Oldest"), lastCrawlTime_desc: msg("Newest Crawl"), lastCrawlTime_asc: msg("Oldest Crawl"), }; /** * Usage: * ```ts * * ``` */ @localized() export class WorkflowsList extends LiteElement { @property({ type: Object }) authState!: AuthState; @property({ type: String }) orgId!: string; @property({ type: String }) userId!: string; @property({ type: Boolean }) isCrawler!: boolean; @state() crawlConfigs?: Workflow[]; @state() runningCrawlsMap: RunningCrawlsMap = {}; @state() showEditDialog?: boolean = false; @state() selectedTemplateForEdit?: Workflow; @state() fetchErrorStatusCode?: number; @state() private orderBy: { field: "created"; direction: "asc" | "desc"; } = { field: "created", direction: "desc", }; @state() private filterByCurrentUser = false; @state() private searchBy: string = ""; @state() private filterByScheduled: boolean | null = null; // For fuzzy search: private fuse = new Fuse([], { keys: ["name", "config.seeds", "config.seeds.url"], shouldSort: false, threshold: 0.2, // stricter; default is 0.6 }); constructor() { super(); this.filterByCurrentUser = window.sessionStorage.getItem(FILTER_BY_CURRENT_USER_STORAGE_KEY) === "true"; } protected async willUpdate(changedProperties: Map) { if ( changedProperties.has("orgId") || changedProperties.has("filterByCurrentUser") ) { this.crawlConfigs = await this.fetchWorkflows(); // Update search/filter collection this.fuse.setCollection(this.crawlConfigs as any); } if (changedProperties.has("filterByCurrentUser")) { window.sessionStorage.setItem( FILTER_BY_CURRENT_USER_STORAGE_KEY, this.filterByCurrentUser.toString() ); } } private async fetchWorkflows() { this.fetchErrorStatusCode = undefined; try { return await this.getWorkflows(); } catch (e: any) { if (e.isApiError) { this.fetchErrorStatusCode = e.statusCode; } else { this.notify({ message: msg("Sorry, couldn't retrieve Workflows at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } render() { return html`

${msg("Workflows")}

${when( this.isCrawler, () => html` ${msg("New Workflow")} ` )}
${this.renderControls()}
${when( this.fetchErrorStatusCode, () => html`
${msg( `Something unexpected went wrong while retrieving Workflows.` )}
`, () => this.crawlConfigs ? this.crawlConfigs.length ? this.renderTemplateList() : html`

${msg("No Workflows yet.")}

` : html`
` )} (this.showEditDialog = false)} @sl-after-hide=${() => (this.selectedTemplateForEdit = undefined)} >

${this.selectedTemplateForEdit?.name}

${this.selectedTemplateForEdit ? html` ` : ""}
`; } private renderControls() { return html`
${msg("Sort By")}
{ const [field, direction] = e.detail.item.value.split("_"); this.orderBy = { field: field, direction: direction, }; }} > ${(sortableFieldLabels as any)[this.orderBy.field] || sortableFieldLabels[ `${this.orderBy.field}_${this.orderBy.direction}` ]} ${Object.entries(sortableFieldLabels).map( ([value, label]) => html` ${label} ` )} { this.orderBy = { ...this.orderBy, direction: this.orderBy.direction === "asc" ? "desc" : "asc", }; }} >
`; } private renderTemplateList() { const flowFns = [ orderBy(this.orderBy.field, this.orderBy.direction), map(this.renderTemplateItem.bind(this)), ]; if (this.filterByScheduled === true) { flowFns.unshift(filter(({ schedule }: any) => Boolean(schedule))); } else if (this.filterByScheduled === false) { flowFns.unshift(filter(({ schedule }: any) => !schedule)); } if (this.searchBy.length >= MIN_SEARCH_LENGTH) { flowFns.unshift(this.filterResults); } return html`
${flow(...flowFns)(this.crawlConfigs)}
`; } private renderTemplateItem(crawlConfig: Workflow) { const name = this.renderName(crawlConfig); return html`
${name}
${when(this.isCrawler, () => this.renderCardMenu(crawlConfig))}
${this.renderCardFooter(crawlConfig)}
`; } private renderCardMenu(t: Workflow) { const menuItems: HTMLTemplateResult[] = [ html` `, ]; if (!t.inactive && !this.runningCrawlsMap[t.id]) { menuItems.unshift(html` `); } if (t.crawlCount && !t.inactive) { menuItems.push(html` `); } if (!t.crawlCount) { menuItems.push(html` `); } return html` e.preventDefault()}> `; } private renderCardFooter(t: Workflow) { if (t.inactive) { return ""; } const crawlId = this.runningCrawlsMap[t.id]; if (crawlId) { return html` `; } if (!this.isCrawler) { return ""; } return html`
`; } private renderName(crawlConfig: Workflow) { if (crawlConfig.name) return crawlConfig.name; const { config } = crawlConfig; const firstSeed = config.seeds[0]; let firstSeedURL = typeof firstSeed === "string" ? firstSeed : firstSeed.url; if (config.seeds.length === 1) { return firstSeedURL; } const remainderCount = config.seeds.length - 1; if (remainderCount === 1) { return msg( html`${firstSeed} +${remainderCount} URL` ); } return msg( html`${firstSeed} +${remainderCount} URLs` ); } private onSearchInput = debounce(200)((e: any) => { this.searchBy = e.target.value; }) as any; private filterResults = () => { const results = this.fuse.search(this.searchBy); return results.map(({ item }) => item); }; /** * Fetch Workflows and record running crawls * associated with the Workflows **/ private async getWorkflows(): Promise { const params = this.filterByCurrentUser ? `?userid=${this.userId}` : ""; const data: APIPaginatedList = await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs${params}`, this.authState! ); const runningCrawlsMap: RunningCrawlsMap = {}; data.items.forEach(({ id, currCrawlId }) => { if (currCrawlId) { runningCrawlsMap[id] = currCrawlId; } }); this.runningCrawlsMap = runningCrawlsMap; return data.items; } /** * Create a new template using existing template data */ private async duplicateConfig(workflow: Workflow) { const workflowParams: WorkflowParams = { ...workflow, name: msg(str`${this.renderName(workflow)} Copy`), }; this.navTo( `/orgs/${this.orgId}/workflows?new&jobType=${workflowParams.jobType}`, { workflow: workflowParams, } ); this.notify({ message: msg(str`Copied Workflowuration to new template.`), variant: "success", icon: "check2-circle", }); } private async deactivateTemplate(crawlConfig: Workflow): Promise { try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${crawlConfig.id}`, this.authState!, { method: "DELETE", } ); this.notify({ message: msg( html`Deactivated ${this.renderName(crawlConfig)}.` ), variant: "success", icon: "check2-circle", }); this.crawlConfigs = this.crawlConfigs!.filter( (t) => t.id !== crawlConfig.id ); } catch { this.notify({ message: msg("Sorry, couldn't deactivate Workflow at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async deleteTemplate(crawlConfig: Workflow): Promise { try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${crawlConfig.id}`, this.authState!, { method: "DELETE", } ); this.notify({ message: msg( html`Deleted ${this.renderName(crawlConfig)}.` ), variant: "success", icon: "check2-circle", }); this.crawlConfigs = this.crawlConfigs!.filter( (t) => t.id !== crawlConfig.id ); } catch { this.notify({ message: msg("Sorry, couldn't delete Workflow at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async runNow(crawlConfig: Workflow): Promise { try { const data = await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${crawlConfig.id}/run`, this.authState!, { method: "POST", } ); const crawlId = data.started; this.runningCrawlsMap = { ...this.runningCrawlsMap, [crawlConfig.id]: crawlId, }; this.notify({ message: msg( html`Started crawl from ${this.renderName(crawlConfig)}.
Watch crawl` ), variant: "success", icon: "check2-circle", duration: 8000, }); } catch (e: any) { this.notify({ message: (e.isApiError && e.statusCode === 403 && msg("You do not have permission to run crawls.")) || msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async onSubmitSchedule(event: { detail: { formData: FormData }; }): Promise { if (!this.selectedTemplateForEdit) return; const { formData } = event.detail; const interval = formData.get("scheduleInterval"); let schedule = ""; if (interval) { schedule = getUTCSchedule({ interval: formData.get("scheduleInterval") as any, hour: formData.get("scheduleHour") as any, minute: formData.get("scheduleMinute") as any, period: formData.get("schedulePeriod") as any, }); } const editedTemplateId = this.selectedTemplateForEdit.id; try { await this.apiFetch( `/orgs/${this.orgId}/crawlconfigs/${editedTemplateId}`, this.authState!, { method: "PATCH", body: JSON.stringify({ schedule }), } ); this.crawlConfigs = this.crawlConfigs?.map((t) => t.id === editedTemplateId ? { ...t, schedule, } : t ); this.showEditDialog = false; this.notify({ message: msg("Successfully saved new schedule."), variant: "success", icon: "check2-circle", }); } catch (e: any) { console.error(e); this.notify({ message: msg("Something went wrong, couldn't update schedule."), variant: "danger", icon: "exclamation-octagon", }); } } } customElements.define("btrix-workflows-list", WorkflowsList);