import { state, property, query } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { msg, localized, str } from "@lit/localize"; import { when } from "lit/directives/when.js"; import type { SlCheckbox, SlSelect } from "@shoelace-style/shoelace"; import queryString from "query-string"; import { CopyButton } from "../../components/copy-button"; import { CrawlStatus } from "../../components/crawl-status"; import type { PageChangeEvent } from "../../components/pagination"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import type { Crawl, CrawlState, Workflow, WorkflowParams } from "./types"; import type { APIPaginatedList, APIPaginationQuery } from "../../types/api"; import { isActive, activeCrawlStates } from "../../utils/crawler"; type Crawls = APIPaginatedList & { items: Crawl[]; }; type SearchFields = "name" | "firstSeed"; type SortField = "finished" | "fileSize"; type SortDirection = "asc" | "desc"; const ABORT_REASON_THROTTLE = "throttled"; const INITIAL_PAGE_SIZE = 20; const FILTER_BY_CURRENT_USER_STORAGE_KEY = "btrix.filterByCurrentUser.crawls"; const POLL_INTERVAL_SECONDS = 10; const sortableFields: Record< SortField, { label: string; defaultDirection?: SortDirection } > = { finished: { label: msg("Date Created"), defaultDirection: "desc", }, fileSize: { label: msg("Size"), defaultDirection: "desc", }, }; const finishedCrawlStates: CrawlState[] = [ "complete", "partial_complete", "timed_out", ]; /** * Usage: * ```ts * * ``` */ @localized() export class CrawlsList extends LiteElement { static FieldLabels: Record = { name: msg("Name"), firstSeed: msg("Crawl Start URL"), }; @property({ type: Object }) authState!: AuthState; @property({ type: String }) userId!: string; @property({ type: String }) orgId?: string; @property({ type: Boolean }) orgStorageQuotaReached = false; @property({ type: Boolean }) isCrawler!: boolean; @property({ type: String }) itemType: Crawl["type"] = null; @state() private archivedItems?: Crawls; @state() private searchOptions: any[] = []; @state() private orderBy: { field: SortField; direction: SortDirection; } = { field: "finished", direction: sortableFields["finished"].defaultDirection!, }; @state() private filterByCurrentUser = false; @state() private filterBy: Partial> = {}; @state() private itemToEdit: Crawl | null = null; @state() private isEditingItem = false; @state() private isUploadingArchive = false; @query("#stateSelect") stateSelect?: SlSelect; // For fuzzy search: private searchKeys = ["name", "firstSeed"]; // Use to cancel requests private getArchivedItemsController: AbortController | null = null; private get selectedSearchFilterKey() { return Object.keys(CrawlsList.FieldLabels).find((key) => Boolean((this.filterBy as any)[key]) ); } constructor() { super(); this.filterByCurrentUser = window.sessionStorage.getItem(FILTER_BY_CURRENT_USER_STORAGE_KEY) === "true"; } protected willUpdate(changedProperties: Map) { if ( changedProperties.has("filterByCurrentUser") || changedProperties.has("filterBy") || changedProperties.has("orderBy") || changedProperties.has("itemType") ) { if (changedProperties.has("itemType")) { this.filterBy = {}; this.orderBy = { field: "finished", direction: sortableFields["finished"].defaultDirection!, }; this.archivedItems = undefined; } this.fetchArchivedItems({ page: 1, pageSize: INITIAL_PAGE_SIZE, }); if (changedProperties.has("filterByCurrentUser")) { window.sessionStorage.setItem( FILTER_BY_CURRENT_USER_STORAGE_KEY, this.filterByCurrentUser.toString() ); } } if (changedProperties.has("itemType")) { this.fetchConfigSearchValues(); } } disconnectedCallback(): void { this.cancelInProgressGetArchivedItems(); super.disconnectedCallback(); } render() { const listTypes: { itemType: Crawl["type"]; label: string; icon?: string; }[] = [ { itemType: null, label: msg("All"), }, { itemType: "crawl", icon: "gear-wide-connected", label: msg("Crawls"), }, { itemType: "upload", icon: "upload", label: msg("Uploads"), }, ]; return html`

${msg("Archived Items")}

${when( this.isCrawler, () => html` (this.isUploadingArchive = true)} ?disabled=${this.orgStorageQuotaReached} > ${msg("Upload WACZ")} ` )}
${listTypes.map(({ label, itemType, icon }) => { const isSelected = itemType === this.itemType; return html` ${icon ? html`` : ""} ${label} `; })}
${this.renderControls()}
${when( this.archivedItems, () => { const { items, page, total, pageSize } = this.archivedItems!; return html`
${items.length ? this.renderArchivedItemList() : this.renderEmptyState()}
${when( total > pageSize, () => html`
{ await this.fetchArchivedItems({ page: e.detail.page, }); // Scroll to top of list // TODO once deep-linking is implemented, scroll to top of pushstate this.scrollIntoView({ behavior: "smooth" }); }} >
` )} `; }, () => html`
` )}
${when( this.isCrawler && this.orgId, () => html` (this.isUploadingArchive = false)} @uploaded=${() => { if (this.itemType !== "crawl") { this.fetchArchivedItems({ page: 1, }); } }} > ` )} `; } private renderControls() { const viewPlaceholder = msg("Any"); const viewOptions = finishedCrawlStates; return html`
${this.renderSearch()}
${msg("Status:")}
{ const value = (e.target as SlSelect).value as CrawlState[]; await this.updateComplete; this.filterBy = { ...this.filterBy, state: value, }; }} > ${viewOptions.map(this.renderStatusMenuItem)}
${msg("Sort by:")}
${this.renderSortControl()}
${this.userId ? html`
` : ""} `; } private renderSortControl() { let options = Object.entries(sortableFields).map( ([value, { label }]) => html` ${label} ` ); return html` { const field = (e.target as HTMLSelectElement).value as SortField; this.orderBy = { field: field, direction: sortableFields[field].defaultDirection || this.orderBy.direction, }; }} > ${options} { this.orderBy = { ...this.orderBy, direction: this.orderBy.direction === "asc" ? "desc" : "asc", }; }} > `; } private renderSearch() { return html` { const { key, value } = e.detail; this.filterBy = { ...this.filterBy, [key]: value, }; }} @on-clear=${() => { const { name, firstSeed, ...otherFilters } = this.filterBy; this.filterBy = otherFilters; }} > `; } private renderArchivedItemList() { if (!this.archivedItems) return; return html` ${this.archivedItems.items.map(this.renderArchivedItem)} (this.isEditingItem = false)} @updated=${ /* TODO fetch current page or single crawl */ this.fetchArchivedItems } > `; } private renderArchivedItem = (item: Crawl) => html` ${when( this.isCrawler, this.crawlerMenuItemsRenderer(item), () => html` this.navTo( `/orgs/${item.oid}/crawls/${ item.type === "upload" ? "upload" : "crawl" }/${item.id}` )} > ${msg("View Crawl Details")} ` )} `; private crawlerMenuItemsRenderer = (item: Crawl) => () => // HACK shoelace doesn't current have a way to override non-hover // color without resetting the --sl-color-neutral-700 variable html` ${when( this.isCrawler, () => html` { this.itemToEdit = item; this.isEditingItem = true; }} > ${msg("Edit Metadata")} ` )} ${when( item.type === "crawl", () => html` this.navTo(`/orgs/${item.oid}/workflows/crawl/${item.cid}`)} > ${msg("Go to Workflow")} CopyButton.copyToClipboard(item.cid)}> ${msg("Copy Workflow ID")} CopyButton.copyToClipboard(item.id)}> ${msg("Copy Crawl ID")} ` )} CopyButton.copyToClipboard(item.tags.join(", "))} ?disabled=${!item.tags.length} > ${msg("Copy Tags")} ${when( this.isCrawler && !isActive(item.state), () => html` this.deleteItem(item)} > ${msg("Delete Item")} ` )} `; private renderStatusMenuItem = (state: CrawlState) => { const { icon, label } = CrawlStatus.getContent(state); return html`${icon}${label}`; }; private renderEmptyState() { if (Object.keys(this.filterBy).length) { return html`

${msg("No matching items found.")}

`; } if (this.archivedItems?.page && this.archivedItems?.page > 1) { return html`

${msg("Could not find page.")}

`; } return html`

${msg("No archived items yet.")}

`; } /** * Fetch archived items and update internal state */ private async fetchArchivedItems(params?: APIPaginationQuery): Promise { this.cancelInProgressGetArchivedItems(); try { this.archivedItems = await this.getArchivedItems(params); } catch (e: any) { if (e.name === "AbortError") { console.debug("Fetch archived items aborted to throttle"); } else { this.notify({ message: msg("Sorry, couldn't retrieve archived items at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } private cancelInProgressGetArchivedItems() { if (this.getArchivedItemsController) { this.getArchivedItemsController.abort(ABORT_REASON_THROTTLE); this.getArchivedItemsController = null; } } private async getArchivedItems( queryParams?: APIPaginationQuery & { state?: CrawlState[] } ): Promise { const query = queryString.stringify( { ...this.filterBy, state: this.filterBy.state?.length ? this.filterBy.state : finishedCrawlStates, page: queryParams?.page || this.archivedItems?.page || 1, pageSize: queryParams?.pageSize || this.archivedItems?.pageSize || INITIAL_PAGE_SIZE, userid: this.filterByCurrentUser ? this.userId : undefined, sortBy: this.orderBy.field, sortDirection: this.orderBy.direction === "desc" ? -1 : 1, crawlType: this.itemType, }, { arrayFormat: "comma", } ); this.getArchivedItemsController = new AbortController(); const data = await this.apiFetch( `/orgs/${this.orgId}/all-crawls?${query}`, this.authState!, { signal: this.getArchivedItemsController.signal, } ); this.getArchivedItemsController = null; return data; } private async fetchConfigSearchValues() { try { const query = queryString.stringify({ crawlType: this.itemType, }); const data: { crawlIds: string[]; names: string[]; descriptions: string[]; firstSeeds: string[]; } = await this.apiFetch( `/orgs/${this.orgId}/all-crawls/search-values?${query}`, this.authState! ); // Update search/filter collection const toSearchItem = (key: SearchFields) => (value: string) => ({ [key]: value, }); this.searchOptions = [ ...data.names.map(toSearchItem("name")), ...data.firstSeeds.map(toSearchItem("firstSeed")), ]; } catch (e) { console.debug(e); } } private async deleteItem(item: Crawl) { if ( !window.confirm(msg(str`Are you sure you want to delete ${item.name}?`)) ) { return; } let apiPath; switch (this.itemType) { case "crawl": apiPath = "crawls"; break; case "upload": apiPath = "uploads"; break; default: apiPath = "all-crawls"; break; } try { const data = await this.apiFetch( `/orgs/${item.oid}/${apiPath}/delete`, this.authState!, { method: "POST", body: JSON.stringify({ crawl_ids: [item.id], }), } ); const { items, ...crawlsData } = this.archivedItems!; this.archivedItems = { ...crawlsData, items: items.filter((c) => c.id !== item.id), }; this.notify({ message: msg(str`Successfully deleted archived item.`), variant: "success", icon: "check2-circle", }); this.fetchArchivedItems(); } catch (e: any) { let message = msg( str`Sorry, couldn't delete archived item at this time.` ); if (e.isApiError) { if (e.details == "not_allowed") { message = msg( str`Only org owners can delete other users' archived items.` ); } else if (e.message) { message = e.message; } } this.notify({ message: message, variant: "danger", icon: "exclamation-octagon", }); } } async getWorkflow(crawl: Crawl): Promise { const data: Workflow = await this.apiFetch( `/orgs/${crawl.oid}/crawlconfigs/${crawl.cid}`, this.authState! ); return data; } } customElements.define("btrix-crawls-list", CrawlsList);