import type { HTMLTemplateResult, PropertyValueMap } from "lit"; import { state, property } from "lit/decorators.js"; import { msg, localized, str } from "@lit/localize"; import { parseCron } from "@cheap-glitch/mi-cron"; import debounce from "lodash/fp/debounce"; import flow from "lodash/fp/flow"; import map from "lodash/fp/map"; import orderBy from "lodash/fp/orderBy"; import filter from "lodash/fp/filter"; import Fuse from "fuse.js"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import type { CrawlConfig, InitialCrawlConfig } from "./types"; import { getUTCSchedule, humanizeNextDate, humanizeSchedule, } from "../../utils/cron"; import "../../components/crawl-scheduler"; import { SlCheckbox } from "@shoelace-style/shoelace"; type RunningCrawlsMap = { /** Map of configId: crawlId */ [configId: string]: string; }; const MIN_SEARCH_LENGTH = 2; const sortableFieldLabels = { created_desc: msg("Newest"), created_asc: msg("Oldest"), lastCrawlTime_desc: msg("Newest Crawl"), lastCrawlTime_asc: msg("Oldest Crawl"), }; /** * Usage: * ```ts * * ``` */ @localized() export class CrawlTemplatesList extends LiteElement { @property({ type: Object }) authState!: AuthState; @property({ type: String }) archiveId!: string; @property({ type: String }) userId!: string; @state() crawlTemplates?: CrawlConfig[]; @state() runningCrawlsMap: RunningCrawlsMap = {}; @state() showEditDialog?: boolean = false; @state() selectedTemplateForEdit?: CrawlConfig; @state() private orderBy: { field: "created"; direction: "asc" | "desc"; } = { field: "created", direction: "desc", }; @state() private filterByCurrentUser = true; @state() private searchBy: string = ""; @state() private filterByScheduled: boolean | null = null; // For fuzzy search: private fuse = new Fuse([], { keys: ["name"], shouldSort: false, threshold: 0.4, // stricter; default is 0.6 }); protected async willUpdate(changedProperties: Map) { if ( changedProperties.has("archiveId") || changedProperties.has("filterByCurrentUser") ) { try { this.crawlTemplates = await this.getCrawlTemplates(); // Update search/filter collection this.fuse.setCollection(this.crawlTemplates as any); } catch (e) { this.notify({ message: msg("Sorry, couldn't retrieve crawl configs at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } render() { return html`
${this.renderControls()}
${this.crawlTemplates ? this.crawlTemplates.length ? this.renderTemplateList() : html`

${msg("No crawl configs yet.")}

` : html`
`} (this.showEditDialog = false)} @sl-after-hide=${() => (this.selectedTemplateForEdit = undefined)} >

${this.selectedTemplateForEdit?.name}

${this.selectedTemplateForEdit ? html` ` : ""}
`; } private renderControls() { return html`
${msg("New Crawl Config")}
${this.userId ? html`` : ""}
${msg("Sort By")}
{ const [field, direction] = e.detail.item.value.split("_"); this.orderBy = { field: field, direction: direction, }; }} > ${(sortableFieldLabels as any)[this.orderBy.field] || sortableFieldLabels[ `${this.orderBy.field}_${this.orderBy.direction}` ]} ${Object.entries(sortableFieldLabels).map( ([value, label]) => html` ${label} ` )} { this.orderBy = { ...this.orderBy, direction: this.orderBy.direction === "asc" ? "desc" : "asc", }; }} >
`; } private renderTemplateList() { const flowFns = [ orderBy(this.orderBy.field, this.orderBy.direction), map(this.renderTemplateItem.bind(this)), ]; if (this.filterByScheduled === true) { flowFns.unshift(filter(({ schedule }: any) => Boolean(schedule))); } else if (this.filterByScheduled === false) { flowFns.unshift(filter(({ schedule }: any) => !schedule)); } if (this.searchBy.length >= MIN_SEARCH_LENGTH) { flowFns.unshift(this.filterResults); } return html`
${flow(...flowFns)(this.crawlTemplates)}
`; } private renderTemplateItem(t: CrawlConfig) { return html`
${t.name}
${this.renderCardMenu(t)}
${this.renderCardFooter(t)}
`; } private renderCardMenu(t: CrawlConfig) { const menuItems: HTMLTemplateResult[] = [ html` `, ]; if (!t.inactive) { menuItems.unshift(html` `); } if (t.crawlCount && !t.inactive) { menuItems.push(html` `); } if (!t.crawlCount) { menuItems.push(html` `); } return html` e.preventDefault()}> `; } private renderCardFooter(t: CrawlConfig) { if (t.inactive) { return ""; } return html`
`; } private onSearchInput = debounce(200)((e: any) => { this.searchBy = e.target.value; }) as any; private filterResults = () => { const results = this.fuse.search(this.searchBy); return results.map(({ item }) => item); }; /** * Fetch crawl configs and record running crawls * associated with the crawl configs **/ private async getCrawlTemplates(): Promise { const params = this.userId && this.filterByCurrentUser ? `?userid=${this.userId}` : ""; const data: { crawlConfigs: CrawlConfig[] } = await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs${params}`, this.authState! ); const runningCrawlsMap: RunningCrawlsMap = {}; data.crawlConfigs.forEach(({ id, currCrawlId }) => { if (currCrawlId) { runningCrawlsMap[id] = currCrawlId; } }); this.runningCrawlsMap = runningCrawlsMap; return data.crawlConfigs; } /** * Create a new template using existing template data */ private async duplicateConfig(template: CrawlConfig) { const crawlTemplate: InitialCrawlConfig = { name: msg(str`${template.name} Copy`), config: template.config, profileid: template.profileid || null, jobType: template.jobType, schedule: template.schedule, tags: template.tags, }; this.navTo(`/archives/${this.archiveId}/crawl-templates/new`, { crawlTemplate, }); this.notify({ message: msg(str`Copied crawl configuration to new template.`), variant: "success", icon: "check2-circle", }); } private async deactivateTemplate(template: CrawlConfig): Promise { try { await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/${template.id}`, this.authState!, { method: "DELETE", } ); this.notify({ message: msg(html`Deactivated ${template.name}.`), variant: "success", icon: "check2-circle", }); this.crawlTemplates = this.crawlTemplates!.filter( (t) => t.id !== template.id ); } catch { this.notify({ message: msg("Sorry, couldn't deactivate crawl config at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async deleteTemplate(template: CrawlConfig): Promise { try { await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/${template.id}`, this.authState!, { method: "DELETE", } ); this.notify({ message: msg(html`Deleted ${template.name}.`), variant: "success", icon: "check2-circle", }); this.crawlTemplates = this.crawlTemplates!.filter( (t) => t.id !== template.id ); } catch { this.notify({ message: msg("Sorry, couldn't delete crawl config at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async runNow(template: CrawlConfig): Promise { try { const data = await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/${template.id}/run`, this.authState!, { method: "POST", } ); const crawlId = data.started; this.runningCrawlsMap = { ...this.runningCrawlsMap, [template.id]: crawlId, }; this.notify({ message: msg( html`Started crawl from ${template.name}.
Watch crawl` ), variant: "success", icon: "check2-circle", duration: 8000, }); } catch { this.notify({ message: msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async onSubmitSchedule(event: { detail: { formData: FormData }; }): Promise { if (!this.selectedTemplateForEdit) return; const { formData } = event.detail; const interval = formData.get("scheduleInterval"); let schedule = ""; if (interval) { schedule = getUTCSchedule({ interval: formData.get("scheduleInterval") as any, hour: formData.get("scheduleHour") as any, minute: formData.get("scheduleMinute") as any, period: formData.get("schedulePeriod") as any, }); } const editedTemplateId = this.selectedTemplateForEdit.id; try { await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/${editedTemplateId}`, this.authState!, { method: "PATCH", body: JSON.stringify({ schedule }), } ); this.crawlTemplates = this.crawlTemplates?.map((t) => t.id === editedTemplateId ? { ...t, schedule, } : t ); this.showEditDialog = false; this.notify({ message: msg("Successfully saved new schedule."), variant: "success", icon: "check2-circle", }); } catch (e: any) { console.error(e); this.notify({ message: msg("Something went wrong, couldn't update schedule."), variant: "danger", icon: "exclamation-octagon", }); } } } customElements.define("btrix-crawl-templates-list", CrawlTemplatesList);