import { state, property, customElement } from "lit/decorators.js"; import { when } from "lit/directives/when.js"; import { msg, localized } from "@lit/localize"; import type { SlSelect } from "@shoelace-style/shoelace"; import queryString from "query-string"; import type { PageChangeEvent } from "@/components/ui/pagination"; import { CrawlStatus } from "@/features/archived-items/crawl-status"; import type { AuthState } from "@/utils/AuthService"; import LiteElement, { html } from "@/utils/LiteElement"; import { needLogin } from "@/utils/auth"; import { activeCrawlStates } from "@/utils/crawler"; import type { Crawl, CrawlState } from "@/types/crawler"; import type { APIPaginationQuery, APIPaginatedList } from "@/types/api"; import "./org/workflow-detail"; import "./org/crawls-list"; type SortField = "started" | "firstSeed" | "fileSize"; type SortDirection = "asc" | "desc"; const sortableFields: Record< SortField, { label: string; defaultDirection?: SortDirection } > = { started: { label: msg("Date Started"), defaultDirection: "desc", }, firstSeed: { label: msg("Crawl Start URL"), defaultDirection: "desc", }, fileSize: { label: msg("File Size"), defaultDirection: "desc", }, }; const ABORT_REASON_THROTTLE = "throttled"; @localized() @customElement("btrix-crawls") @needLogin export class Crawls extends LiteElement { @property({ type: Object }) authState!: AuthState; @property({ type: String }) crawlId?: string; @state() private crawl?: Crawl; @state() private crawls?: APIPaginatedList; @state() private slugLookup: Record = {}; @state() private orderBy: { field: SortField; direction: SortDirection; } = { field: "started", direction: sortableFields["started"].defaultDirection!, }; @state() private filterBy: Partial> = { state: activeCrawlStates, }; // Use to cancel requests private getCrawlsController: AbortController | null = null; protected async willUpdate(changedProperties: Map) { if (changedProperties.has("crawlId") && this.crawlId) { // Redirect to org crawl page await this.fetchWorkflowId(); const slug = this.slugLookup[this.crawl!.oid]; this.navTo(`/orgs/${slug}/items/crawl/${this.crawlId}`); } else { if ( changedProperties.has("filterBy") || changedProperties.has("orderBy") ) { this.fetchCrawls(); } } } firstUpdated() { this.fetchSlugLookup(); } disconnectedCallback(): void { this.cancelInProgressGetCrawls(); super.disconnectedCallback(); } render() { return html`
${this.crawlId ? // Render loading indicator while preparing to redirect this.renderLoading() : this.renderCrawls()}
`; } private renderCrawls() { return html`

${msg("All Running Crawls")}

${this.renderControls()}
${when( this.crawls, () => { const { items, page, total, pageSize } = this.crawls!; const hasCrawlItems = items.length; return html`
${hasCrawlItems ? this.renderCrawlList() : this.renderEmptyState()}
${when( hasCrawlItems || page > 1, () => html`
{ await this.fetchCrawls({ page: e.detail.page, }); // Scroll to top of list // TODO once deep-linking is implemented, scroll to top of pushstate this.scrollIntoView({ behavior: "smooth" }); }} >
` )} `; }, this.renderLoading )}
`; } private renderLoading = () => html`
`; private renderControls() { const viewPlaceholder = msg("Any Active Status"); const viewOptions = activeCrawlStates; return html`
${msg("View:")}
{ const value = (e.target as SlSelect).value as CrawlState[]; await this.updateComplete; this.filterBy = { ...this.filterBy, state: value, }; }} > ${viewOptions.map(this.renderStatusMenuItem)}
${msg("Sort by:")}
${this.renderSortControl()}
`; } private renderSortControl() { const options = Object.entries(sortableFields).map( ([value, { label }]) => html` ${label} ` ); return html` { const field = (e.target as HTMLSelectElement).value as SortField; this.orderBy = { field: field, direction: sortableFields[field].defaultDirection || this.orderBy.direction, }; }} > ${options} { this.orderBy = { ...this.orderBy, direction: this.orderBy.direction === "asc" ? "desc" : "asc", }; }} > `; } private renderStatusMenuItem = (state: CrawlState) => { const { icon, label } = CrawlStatus.getContent(state); return html`${icon}${label}`; }; private renderCrawlList() { if (!this.crawls) return; return html` ${this.crawls.items.map(this.renderCrawlItem)} `; } private renderEmptyState() { if (this.crawls?.page && this.crawls?.page > 1) { return html`

${msg("Could not find page.")}

`; } return html`

${msg("No matching crawls found.")}

`; } private renderCrawlItem = (crawl: Crawl) => html` this.navTo(`/crawls/crawl/${crawl.id}#settings`)} > ${msg("View Crawl Settings")} `; private async fetchWorkflowId() { try { this.crawl = await this.getCrawl(); } catch (e) { console.error(e); } } private async fetchSlugLookup() { try { this.slugLookup = await this.getSlugLookup(); } catch (e: any) { console.debug(e); } } /** * Fetch crawls and update internal state */ private async fetchCrawls(params?: APIPaginationQuery): Promise { this.cancelInProgressGetCrawls(); try { this.crawls = await this.getCrawls(params); } catch (e: any) { if (e.name === "AbortError") { console.debug("Fetch crawls aborted to throttle"); } else { this.notify({ message: msg("Sorry, couldn't retrieve crawls at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private cancelInProgressGetCrawls() { if (this.getCrawlsController) { this.getCrawlsController.abort(ABORT_REASON_THROTTLE); this.getCrawlsController = null; } } private async getCrawls( queryParams?: APIPaginationQuery & { state?: CrawlState[] } ) { const query = queryString.stringify( { ...this.filterBy, ...queryParams, page: queryParams?.page || this.crawls?.page || 1, pageSize: queryParams?.pageSize || this.crawls?.pageSize || 100, sortBy: this.orderBy.field, sortDirection: this.orderBy.direction === "desc" ? -1 : 1, }, { arrayFormat: "comma", } ); this.getCrawlsController = new AbortController(); const data = await this.apiFetch>( `/orgs/all/crawls?${query}`, this.authState!, { signal: this.getCrawlsController.signal, } ); this.getCrawlsController = null; return data; } private async getCrawl() { const data: Crawl = await this.apiFetch( `/orgs/all/crawls/${this.crawlId}/replay.json`, this.authState! ); return data; } private async getSlugLookup() { const data = await this.apiFetch>( `/orgs/slug-lookup`, this.authState! ); return data; } }