import { state, property } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { msg, localized, str } from "@lit/localize"; import debounce from "lodash/fp/debounce"; import flow from "lodash/fp/flow"; import map from "lodash/fp/map"; import orderBy from "lodash/fp/orderBy"; import Fuse from "fuse.js"; import { CopyButton } from "../../components/copy-button"; import { RelativeDuration } from "../../components/relative-duration"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import type { Crawl, CrawlConfig, InitialCrawlConfig } from "./types"; import { SlCheckbox } from "@shoelace-style/shoelace"; type CrawlSearchResult = { item: Crawl; }; const POLL_INTERVAL_SECONDS = 10; const MIN_SEARCH_LENGTH = 2; const sortableFieldLabels = { started_desc: msg("Newest"), started_asc: msg("Oldest"), finished_desc: msg("Recently Updated"), finished_asc: msg("Oldest Finished"), state: msg("Status"), configName: msg("Crawl Name"), cid: msg("Crawl Config ID"), fileSize_asc: msg("Smallest Files"), fileSize_desc: msg("Largest Files"), }; function isActive(crawl: Crawl) { return ( crawl.state === "running" || crawl.state === "starting" || crawl.state === "stopping" ); } /** * Usage: * ```ts * * ``` */ @localized() export class CrawlsList extends LiteElement { @property({ type: Object }) authState!: AuthState; @property({ type: String }) userId!: string; // e.g. `/org/${this.orgId}/crawls` @property({ type: String }) crawlsBaseUrl!: string; // e.g. `/org/${this.orgId}/crawls` @property({ type: String }) crawlsAPIBaseUrl?: string; /** * Fetch & refetch data when needed, * e.g. when component is visible **/ @property({ type: Boolean }) shouldFetch?: boolean; @state() private lastFetched?: number; @state() private crawls?: Crawl[]; @state() private orderBy: { field: "started"; direction: "asc" | "desc"; } = { field: "started", direction: "desc", }; @state() private filterByCurrentUser = true; @state() private filterBy: string = ""; // For fuzzy search: private fuse = new Fuse([], { keys: ["cid", "configName"], shouldSort: false, }); private timerId?: number; // TODO localize private numberFormatter = new Intl.NumberFormat(); private sortCrawls(crawls: CrawlSearchResult[]): CrawlSearchResult[] { return orderBy(({ item }) => item[this.orderBy.field])( this.orderBy.direction )(crawls) as CrawlSearchResult[]; } protected willUpdate(changedProperties: Map) { if ( changedProperties.has("shouldFetch") || changedProperties.get("crawlsBaseUrl") || changedProperties.get("crawlsAPIBaseUrl") || changedProperties.has("filterByCurrentUser") ) { if (this.shouldFetch) { if (!this.crawlsBaseUrl) { throw new Error("Crawls base URL not defined"); } this.fetchCrawls(); } else { this.stopPollTimer(); } } } disconnectedCallback(): void { this.stopPollTimer(); super.disconnectedCallback(); } render() { if (!this.crawls) { return html`
`; } return html`
${this.renderControls()}
${this.crawls.length ? this.renderCrawlList() : html`

${msg("No crawls yet.")}

`}
${this.lastFetched ? msg(html`Last updated: `) : ""}
`; } private renderControls() { return html`
${this.userId ? html`` : ""}
${msg("Sort By")}
{ const [field, direction] = e.detail.item.value.split("_"); this.orderBy = { field: field, direction: direction, }; }} > ${(sortableFieldLabels as any)[this.orderBy.field] || sortableFieldLabels[ `${this.orderBy.field}_${this.orderBy.direction}` ]} ${Object.entries(sortableFieldLabels).map( ([value, label]) => html` ${label} ` )} { this.orderBy = { ...this.orderBy, direction: this.orderBy.direction === "asc" ? "desc" : "asc", }; }} >
`; } private renderCrawlList() { // Return search results if valid filter string is available, // otherwise format crawls list like search results const filterResults = this.filterBy.length >= MIN_SEARCH_LENGTH ? () => this.fuse.search(this.filterBy) : map((crawl) => ({ item: crawl })); return html` `; } private renderCrawlItem = ({ item: crawl }: CrawlSearchResult) => { return html`
  • ${crawl.configName || crawl.cid}
    e.preventDefault()} hoist>
    ${crawl.state.replace(/_/g, " ")}
    ${crawl.finished ? html` ` : ""} ${!crawl.finished ? html` ${crawl.state === "canceled" ? msg("Unknown") : ""} ${isActive(crawl) ? this.renderActiveDuration(crawl) : ""} ` : ""}
    ${crawl.finished ? html`
    (${crawl.fileCount === 1 ? msg(str`${crawl.fileCount} file`) : msg(str`${crawl.fileCount} files`)})
    ${msg( str`in ${RelativeDuration.humanize( new Date(`${crawl.finished}Z`).valueOf() - new Date(`${crawl.started}Z`).valueOf(), { compact: true } )}` )}
    ` : crawl.stats ? html`
    ${this.numberFormatter.format(+crawl.stats.done)} / ${this.numberFormatter.format(+crawl.stats.found)}
    ${msg("pages crawled")}
    ` : ""}
    ${crawl.manual ? html`
    ${msg("Manual Start")}
    ${msg(str`by ${crawl.userName || crawl.userid}`)}
    ` : html`
    ${msg("Scheduled Run")}
    `}
  • `; }; private renderActiveDuration(crawl: Crawl) { const endTime = this.lastFetched || Date.now(); const duration = endTime - new Date(`${crawl.started}Z`).valueOf(); let unitCount: number; let tickSeconds: number | undefined = undefined; // Show second unit if showing seconds or greater than 1 hr const showSeconds = duration < 60 * 2 * 1000; if (showSeconds || duration > 60 * 60 * 1000) { unitCount = 2; } else { unitCount = 1; } // Tick if seconds are showing if (showSeconds) { tickSeconds = 1; } else { tickSeconds = undefined; } return html` `; } private onSearchInput = debounce(200)((e: any) => { this.filterBy = e.target.value; }) as any; /** * Fetch crawls and update internal state */ private async fetchCrawls(): Promise { if (!this.shouldFetch) return; try { const { crawls } = await this.getCrawls(); this.crawls = crawls; // Update search/filter collection this.fuse.setCollection(this.crawls as any); // Restart timer for next poll this.stopPollTimer(); this.timerId = window.setTimeout(() => { this.fetchCrawls(); }, 1000 * POLL_INTERVAL_SECONDS); } catch (e) { this.notify({ message: msg("Sorry, couldn't retrieve crawls at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private stopPollTimer() { window.clearTimeout(this.timerId); } private async getCrawls(): Promise<{ crawls: Crawl[] }> { const params = this.userId && this.filterByCurrentUser ? `?userid=${this.userId}` : ""; const data = await this.apiFetch( `${this.crawlsAPIBaseUrl || this.crawlsBaseUrl}${params}`, this.authState! ); this.lastFetched = Date.now(); return data; } private async cancel(crawl: Crawl) { if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) { const data = await this.apiFetch( `/orgs/${crawl.oid}/crawls/${crawl.id}/cancel`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchCrawls(); } else { this.notify({ message: msg("Something went wrong, couldn't cancel crawl."), variant: "danger", icon: "exclamation-octagon", }); } } } private async stop(crawl: Crawl) { if (window.confirm(msg("Are you sure you want to stop the crawl?"))) { const data = await this.apiFetch( `/orgs/${crawl.oid}/crawls/${crawl.id}/stop`, this.authState!, { method: "POST", } ); if (data.success === true) { this.fetchCrawls(); } else { this.notify({ message: msg("Something went wrong, couldn't stop crawl."), variant: "danger", icon: "exclamation-octagon", }); } } } private async runNow(crawl: Crawl) { // Get crawl config to check if crawl is already running const crawlTemplate = await this.getCrawlTemplate(crawl); if (crawlTemplate?.currCrawlId) { this.notify({ message: msg( html`Crawl of ${crawl.configName} is already running.
    View crawl` ), variant: "warning", icon: "exclamation-triangle", }); return; } try { const data = await this.apiFetch( `/orgs/${crawl.oid}/crawlconfigs/${crawl.cid}/run`, this.authState!, { method: "POST", } ); if (data.started) { this.fetchCrawls(); } this.notify({ message: msg( html`Started crawl from ${crawl.configName}.
    Watch crawl` ), variant: "success", icon: "check2-circle", duration: 8000, }); } catch (e: any) { if (e.isApiError && e.statusCode === 404) { this.notify({ message: msg( html`Sorry, cannot rerun crawl from a deactivated crawl config.
    ` ), variant: "danger", icon: "exclamation-octagon", duration: 8000, }); } else { this.notify({ message: msg("Sorry, couldn't run crawl at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } async getCrawlTemplate(crawl: Crawl): Promise { const data: CrawlConfig = await this.apiFetch( `/orgs/${crawl.oid}/crawlconfigs/${crawl.cid}`, this.authState! ); return data; } /** * Create a new template using existing template data */ private async duplicateConfig(crawl: Crawl, template: CrawlConfig) { const crawlTemplate: InitialCrawlConfig = { name: msg(str`${template.name} Copy`), config: template.config, profileid: template.profileid || null, jobType: template.jobType, schedule: template.schedule, tags: template.tags, }; this.navTo(`/orgs/${crawl.oid}/crawl-templates/new`, { crawlTemplate, }); this.notify({ message: msg(str`Copied crawl configuration to new template.`), variant: "success", icon: "check2-circle", }); } } customElements.define("btrix-crawls-list", CrawlsList);