import { localized, msg, str } from "@lit/localize"; import type { SlDialog, SlSelectEvent } from "@shoelace-style/shoelace"; import clsx from "clsx"; import { html, type PropertyValues } from "lit"; import { customElement, query, state } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { when } from "lit/directives/when.js"; import queryString from "query-string"; import { ScopeType, type ListWorkflow, type Seed, type Workflow, } from "./types"; import { BtrixElement } from "@/classes/BtrixElement"; import type { BtrixFilterChipChangeEvent, FilterChip, } from "@/components/ui/filter-chip"; import { parsePage, type PageChangeEvent } from "@/components/ui/pagination"; import { type SelectEvent } from "@/components/ui/search-combobox"; import { ClipboardController } from "@/controllers/clipboard"; import { SearchParamsController } from "@/controllers/searchParams"; import type { SelectJobTypeEvent } from "@/features/crawl-workflows/new-workflow-dialog"; import { type BtrixChangeWorkflowProfileFilterEvent } from "@/features/crawl-workflows/workflow-profile-filter"; import type { BtrixChangeWorkflowScheduleFilterEvent } from "@/features/crawl-workflows/workflow-schedule-filter"; import type { BtrixChangeWorkflowTagFilterEvent } from "@/features/crawl-workflows/workflow-tag-filter"; import { pageHeader } from "@/layouts/pageHeader"; import { WorkflowTab } from "@/routes"; import scopeTypeLabels from "@/strings/crawl-workflows/scopeType"; import { deleteConfirmation } from "@/strings/ui"; import type { APIPaginatedList, APIPaginationQuery } from "@/types/api"; import { NewWorkflowOnlyScopeType, type StorageSeedFile, } from "@/types/workflow"; import { isApiError } from "@/utils/api"; import { settingsForDuplicate } from "@/utils/crawl-workflows/settingsForDuplicate"; import { isArchivingDisabled } from "@/utils/orgs"; import { tw } from "@/utils/tailwind"; type SearchFields = "name" | "firstSeed"; type SortField = "lastRun" | "name" | "firstSeed" | "created" | "modified"; const SORT_DIRECTIONS = ["asc", "desc"] as const; type SortDirection = (typeof SORT_DIRECTIONS)[number]; type Sort = { field: SortField; direction: SortDirection; }; const FILTER_BY_CURRENT_USER_STORAGE_KEY = "btrix.filterByCurrentUser.crawlConfigs"; const INITIAL_PAGE_SIZE = 10; const POLL_INTERVAL_SECONDS = 10; const ABORT_REASON_THROTTLE = "throttled"; const sortableFields: Record< SortField, { label: string; defaultDirection?: SortDirection } > = { lastRun: { label: msg("Latest Crawl"), defaultDirection: "desc", }, modified: { label: msg("Last Modified"), defaultDirection: "desc", }, name: { label: msg("Name"), defaultDirection: "asc", }, firstSeed: { label: msg("Crawl Start URL"), defaultDirection: "asc", }, created: { label: msg("Date Created"), defaultDirection: "desc", }, }; const DEFAULT_SORT = { field: "lastRun", direction: sortableFields["lastRun"].defaultDirection!, } as const; const USED_FILTERS = [ "schedule", "isCrawlRunning", ] as const satisfies (keyof ListWorkflow)[]; /** * Usage: * ```ts * * ``` */ @customElement("btrix-workflows-list") @localized() export class WorkflowsList extends BtrixElement { static FieldLabels: Record = { name: msg("Name"), firstSeed: msg("Crawl Start URL"), }; @state() private workflows?: APIPaginatedList; @state() private searchOptions: { [x: string]: string }[] = []; @state() private isFetching = false; @state() private fetchErrorStatusCode?: number; @state() private workflowToDelete?: ListWorkflow; @state() private orderBy: Sort = DEFAULT_SORT; @state() private filterBy: Partial<{ [k in keyof ListWorkflow]: boolean }> = {}; @state() private filterByCurrentUser = false; @state() private filterByTags?: string[]; @state() private filterByTagsType: "and" | "or" = "or"; @state() private filterByProfiles?: string[]; @query("#deleteDialog") private readonly deleteDialog?: SlDialog | null; // For fuzzy search: private readonly searchKeys = ["name", "firstSeed"]; // Use to cancel requests private getWorkflowsController: AbortController | null = null; private timerId?: number; private get selectedSearchFilterKey() { return Object.keys(WorkflowsList.FieldLabels).find((key) => Boolean((this.filterBy as Record)[key]), ); } searchParams = new SearchParamsController(this, (params) => { this.updateFiltersFromSearchParams(params); }); private updateFiltersFromSearchParams( params = this.searchParams.searchParams, ) { const filterBy = { ...this.filterBy }; // remove filters no longer present in search params for (const key of Object.keys(filterBy)) { if (!params.has(key)) { filterBy[key as keyof typeof filterBy] = undefined; } } // remove current user filter if not present in search params if (!params.has("mine")) { this.filterByCurrentUser = false; } if (params.has("tags")) { this.filterByTags = params.getAll("tags"); } else { this.filterByTags = undefined; } if (params.has("profiles")) { this.filterByProfiles = params.getAll("profiles"); } else { this.filterByProfiles = undefined; } // add filters present in search params for (const [key, value] of params) { // Filter by current user if (key === "mine") { this.filterByCurrentUser = value === "true"; } if (key === "tagsType") { this.filterByTagsType = value === "and" ? "and" : "or"; } // Sorting field if (key === "sortBy") { if (value in sortableFields) { this.orderBy = { field: value as SortField, direction: // Use default direction for field if available, otherwise use current direction sortableFields[value as SortField].defaultDirection || this.orderBy.direction, }; } } if (key === "sortDir") { if (SORT_DIRECTIONS.includes(value as SortDirection)) { // Overrides sort direction if specified this.orderBy = { ...this.orderBy, direction: value as SortDirection }; } } // Ignored params if ( [ "page", "mine", "tags", "tagsType", "profiles", "sortBy", "sortDir", ].includes(key) ) continue; // Convert string bools to filter values if (value === "true") { filterBy[key as keyof typeof filterBy] = true; } else if (value === "false") { filterBy[key as keyof typeof filterBy] = false; } else { filterBy[key as keyof typeof filterBy] = undefined; } } this.filterBy = { ...filterBy }; } constructor() { super(); this.updateFiltersFromSearchParams(); } connectedCallback() { super.connectedCallback(); // Apply filterByCurrentUser from session storage, and transparently update url without pushing to history stack // This needs to happen here instead of in the constructor because this only occurs once after the element is connected to the DOM, // and so it overrides the filter state set in `updateFiltersFromSearchParams` but only on first render, not on subsequent navigation. this.filterByCurrentUser = window.sessionStorage.getItem(FILTER_BY_CURRENT_USER_STORAGE_KEY) === "true"; if (this.filterByCurrentUser) { this.searchParams.set("mine", "true", { replace: true }); } } protected async willUpdate( changedProperties: PropertyValues & Map, ) { // Props that reset the page to 1 when changed const resetToFirstPageProps = [ "filterByCurrentUser", "filterByTags", "filterByTagsType", "filterByProfiles", "filterByScheduled", "filterBy", "orderBy", ]; // Props that require a data refetch const refetchDataProps = [...resetToFirstPageProps]; if (refetchDataProps.some((k) => changedProperties.has(k))) { const isInitialRender = resetToFirstPageProps .map((k) => changedProperties.get(k)) .every((v) => v === undefined); void this.fetchWorkflows({ page: // If this is the initial render, use the page from the URL or default to 1; otherwise, reset the page to 1 isInitialRender ? parsePage(new URLSearchParams(location.search).get("page")) || 1 : 1, }); } if (changedProperties.has("filterByCurrentUser")) { window.sessionStorage.setItem( FILTER_BY_CURRENT_USER_STORAGE_KEY, this.filterByCurrentUser.toString(), ); } } protected firstUpdated() { void this.fetchConfigSearchValues(); } protected updated( changedProperties: PropertyValues & Map, ) { if ( changedProperties.has("filterBy") || changedProperties.has("filterByCurrentUser") || changedProperties.has("filterByTags") || changedProperties.has("filterByTagsType") || changedProperties.has("filterByProfiles") || changedProperties.has("orderBy") ) { this.searchParams.update((params) => { // Reset page params.delete("page"); const newParams = [ // Known filters ...USED_FILTERS.map<[string, undefined]>((f) => [f, undefined]), // Existing filters ...Object.entries(this.filterBy), // Filter by current user ["mine", this.filterByCurrentUser || undefined], ["tags", this.filterByTags], [ "tagsType", this.filterByTagsType !== "or" ? this.filterByTagsType : undefined, ], ["profiles", this.filterByProfiles], // Sorting fields [ "sortBy", this.orderBy.field !== DEFAULT_SORT.field ? this.orderBy.field : undefined, ], [ "sortDir", this.orderBy.direction !== sortableFields[this.orderBy.field].defaultDirection ? this.orderBy.direction : undefined, ], ] satisfies [string, boolean | string | string[] | undefined][]; for (const [filter, value] of newParams) { if (value !== undefined) { if (Array.isArray(value)) { // Rather than a more efficient method where we compare the existing & wanted arrays, // it's simpler to just delete and re-append values here. If we were working with large // arrays, we could change this, but we'll leave it as is for now — if we were working // with truly large arrays, we wouldn't be using search params anyways. params.delete(filter); value.forEach((v) => { params.append(filter, v); }); } else { params.set(filter, value.toString()); } } else { params.delete(filter); } } return params; }); } } disconnectedCallback(): void { this.cancelInProgressGetWorkflows(); super.disconnectedCallback(); } private async fetchWorkflows(params?: APIPaginationQuery) { this.fetchErrorStatusCode = undefined; this.cancelInProgressGetWorkflows(); this.isFetching = true; try { const workflows = await this.getWorkflows(params); this.workflows = workflows; } catch (e) { if (isApiError(e)) { this.fetchErrorStatusCode = e.statusCode; } else if ((e as Error).name === "AbortError") { console.debug("Fetch archived items aborted to throttle"); } else { this.notify.toast({ message: msg("Sorry, couldn't retrieve Workflows at this time."), variant: "danger", icon: "exclamation-octagon", id: "workflow-retrieve-error", }); } } this.isFetching = false; // Restart timer for next poll this.timerId = window.setTimeout(() => { void this.fetchWorkflows(); }, 1000 * POLL_INTERVAL_SECONDS); } private cancelInProgressGetWorkflows() { window.clearTimeout(this.timerId); if (this.getWorkflowsController) { this.getWorkflowsController.abort(ABORT_REASON_THROTTLE); this.getWorkflowsController = null; } } render() { return html`
${pageHeader({ title: msg("Crawl Workflows"), actions: html` ${when( this.appState.isAdmin, () => html` `, )} ${when( this.appState.isCrawler, () => html` this.navigate.to( `${this.navigate.orgBasePath}/workflows/new`, { scopeType: this.appState.userPreferences?.newWorkflowScopeType, }, )} > ${msg("New Workflow")} { const { value } = e.detail.item; if (value) { this.dispatchEvent( new CustomEvent( "select-job-type", { detail: value as SelectJobTypeEvent["detail"], }, ), ); } }} > ${msg("Scope options")} ${msg("Page Crawl")} ${scopeTypeLabels[ScopeType.Page]} ${scopeTypeLabels[NewWorkflowOnlyScopeType.PageList]} ${scopeTypeLabels[ScopeType.SPA]} ${msg("Site Crawl")} ${scopeTypeLabels[ScopeType.Prefix]} ${scopeTypeLabels[ScopeType.Host]} ${scopeTypeLabels[ScopeType.Domain]} ${scopeTypeLabels[ScopeType.Custom]} `, )} `, classNames: tw`border-b-transparent`, })}
${this.renderControls()}
${when( this.fetchErrorStatusCode, () => html`
${msg( `Something unexpected went wrong while retrieving Workflows.`, )}
`, () => html`
${this.workflows ? this.workflows.total ? this.renderWorkflowList() : this.renderEmptyState() : this.renderLoading()}
`, )} ${this.renderDialogs()} `; } private renderDialogs() { return html` ${when( this.workflowToDelete, (workflow) => html` ${deleteConfirmation(this.renderName(workflow))}
void this.deleteDialog?.hide()} >${msg("Cancel")} { void this.deleteDialog?.hide(); try { await this.delete(workflow); this.workflowToDelete = undefined; } catch { void this.deleteDialog?.show(); } }} >${msg("Delete Workflow")}
`, )} `; } private renderControls() { return html`
${this.renderSearch()}
{ const field = (e.target as HTMLSelectElement).value as SortField; this.orderBy = { field: field, direction: sortableFields[field].defaultDirection || this.orderBy.direction, }; }} > ${Object.entries(sortableFields).map( ([value, { label }]) => html` ${label} `, )} { this.orderBy = { ...this.orderBy, direction: this.orderBy.direction === "asc" ? "desc" : "asc", }; }} >
${this.renderFilters()}
`; } private renderFilters() { return html`
${msg("Filter by:")} { this.filterBy = { ...this.filterBy, schedule: e.detail.value, }; }} > { this.filterByTags = e.detail.value?.tags; this.filterByTagsType = e.detail.value?.type || "or"; }} > { this.filterByProfiles = e.detail.value; }} > { const { checked } = e.target as FilterChip; this.filterBy = { ...this.filterBy, isCrawlRunning: checked ? true : undefined, }; }} > ${msg("Running")} { const { checked } = e.target as FilterChip; this.filterByCurrentUser = Boolean(checked); }} > ${msg("Mine")} ${when( [ this.filterBy.schedule, this.filterBy.isCrawlRunning, this.filterByCurrentUser || undefined, this.filterByTags, ].filter((v) => v !== undefined).length > 1, () => html` { this.filterBy = { ...this.filterBy, schedule: undefined, isCrawlRunning: undefined, }; this.filterByCurrentUser = false; this.filterByTags = undefined; }} > ${msg("Clear All")} `, )}
`; } private renderSearch() { return html` ) => { const { key, value } = e.detail; if (key == null) return; this.filterBy = { [key]: value, }; }} @btrix-clear=${() => { const { name: _name, firstSeed: _firstSeed, ...otherFilters } = this.filterBy; this.filterBy = otherFilters; }} > `; } private renderWorkflowList() { if (!this.workflows) return; const { page, total, pageSize } = this.workflows; return html` ${this.workflows.items.map(this.renderWorkflowItem)}
{ await this.fetchWorkflows({ page: e.detail.page, }); // Scroll to top of list // TODO once deep-linking is implemented, scroll to top of pushstate this.scrollIntoView({ behavior: "smooth" }); }} >
`; } private readonly renderWorkflowItem = (workflow: ListWorkflow) => html` ${this.renderMenuItems(workflow)} `; private renderMenuItems(workflow: ListWorkflow) { return html` ${when( workflow.isCrawlRunning && this.appState.isCrawler, // HACK shoelace doesn't current have a way to override non-hover // color without resetting the --sl-color-neutral-700 variable () => html` void this.stop(workflow.lastCrawlId)} ?disabled=${workflow.lastCrawlStopping} > ${msg("Stop Crawl")} void this.cancel(workflow.lastCrawlId)} > ${msg(html`Cancel & Discard Crawl`)} `, )} ${when( this.appState.isCrawler && !workflow.isCrawlRunning, () => html` void this.runNow(workflow)} > ${msg("Run Crawl")} `, )} ${when( this.appState.isCrawler && workflow.isCrawlRunning && !workflow.lastCrawlStopping, // HACK shoelace doesn't current have a way to override non-hover // color without resetting the --sl-color-neutral-700 variable () => html` this.navigate.to( `${this.navigate.orgBasePath}/workflows/${workflow.id}/${WorkflowTab.LatestCrawl}`, { dialog: "scale", }, )} > ${msg("Edit Browser Windows")} this.navigate.to( `${this.navigate.orgBasePath}/workflows/${workflow.id}/${WorkflowTab.LatestCrawl}`, { dialog: "exclusions", }, )} > ${msg("Edit Exclusions")} `, )} ${when( this.appState.isCrawler, () => html` this.navigate.to( `${this.navigate.orgBasePath}/workflows/${workflow.id}?edit`, )} > ${msg("Edit Workflow Settings")} `, )} ClipboardController.copyToClipboard(workflow.tags.join(", "))} ?disabled=${!workflow.tags.length} > ${msg("Copy Tags")} ${when( this.appState.isCrawler, () => html` void this.duplicateConfig(workflow)} > ${msg("Duplicate Workflow")} ClipboardController.copyToClipboard(workflow.id)} > ${msg("Copy Workflow ID")} ${when( !workflow.crawlCount, () => html` { this.workflowToDelete = workflow; await this.updateComplete; void this.deleteDialog?.show(); }} > ${msg("Delete Workflow")} `, )} `, )} `; } private renderName(crawlConfig: ListWorkflow) { if (crawlConfig.name) return crawlConfig.name; const { firstSeed, seedCount } = crawlConfig; if (seedCount === 1) { return firstSeed; } const remainderCount = seedCount - 1; if (remainderCount === 1) { return msg( html`${firstSeed} +${remainderCount} URL`, ); } return msg( html`${firstSeed} +${remainderCount} URLs`, ); } private renderEmptyState() { if ( Object.keys(this.filterBy).length || this.filterByCurrentUser || this.filterByTags ) { return html`

${msg("No matching Workflows found.")}

`; } if (this.workflows?.page && this.workflows.page > 1) { return html`

${msg("Could not find page.")}

`; } if (this.isFetching) { return this.renderLoading(); } return html`

${msg("No Workflows yet.")}

`; } private renderLoading() { return html`
`; } /** * Fetch Workflows and update state **/ private async getWorkflows( queryParams?: APIPaginationQuery & Record, ) { const query = queryString.stringify( { ...this.filterBy, page: queryParams?.page || this.workflows?.page || parsePage(new URLSearchParams(location.search).get("page")), pageSize: queryParams?.pageSize || this.workflows?.pageSize || INITIAL_PAGE_SIZE, userid: this.filterByCurrentUser ? this.userInfo?.id : undefined, tag: this.filterByTags || undefined, tagMatch: this.filterByTagsType, profileIds: this.filterByProfiles || undefined, sortBy: this.orderBy.field, sortDirection: this.orderBy.direction === "desc" ? -1 : 1, }, { arrayFormat: "none", // For tags }, ); this.getWorkflowsController = new AbortController(); const data = await this.api.fetch>( `/orgs/${this.orgId}/crawlconfigs?${query}`, { signal: this.getWorkflowsController.signal, }, ); this.getWorkflowsController = null; return data; } /** * Create a new template using existing template data */ private async duplicateConfig(workflow: ListWorkflow) { const fullWorkflow = await this.getWorkflow(workflow); let seeds; let seedFile; if (fullWorkflow.config.seedFileId) { seedFile = await this.getSeedFile(fullWorkflow.config.seedFileId); } else { seeds = await this.getSeeds(workflow); } const settings = settingsForDuplicate({ workflow: fullWorkflow, seeds, seedFile, }); this.navigate.to(`${this.navigate.orgBasePath}/workflows/new`, settings); if (seeds && seeds.total > seeds.items.length) { const urlCount = this.localize.number(seeds.items.length); // This is likely an edge case for old workflows with >1,000 seeds // or URL list workflows created via API. this.notify.toast({ title: msg(str`Partially copied workflow settings`), message: msg(str`The first ${urlCount} URLs were copied.`), variant: "warning", id: "workflow-copied-status", }); } else { this.notify.toast({ message: msg("Copied settings to new workflow."), variant: "success", icon: "check2-circle", id: "workflow-copied-status", }); } } private async delete(workflow: ListWorkflow): Promise { try { await this.api.fetch(`/orgs/${this.orgId}/crawlconfigs/${workflow.id}`, { method: "DELETE", }); void this.fetchWorkflows(); this.notify.toast({ message: msg( html`Deleted ${this.renderName(workflow)} Workflow.`, ), variant: "success", icon: "check2-circle", id: "workflow-delete-status", }); } catch { this.notify.toast({ message: msg("Sorry, couldn't delete Workflow at this time."), variant: "danger", icon: "exclamation-octagon", id: "workflow-delete-status", }); } } private async cancel(crawlId: ListWorkflow["lastCrawlId"]) { if (!crawlId) return; if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) { const data = await this.api.fetch<{ success: boolean }>( `/orgs/${this.orgId}/crawls/${crawlId}/cancel`, { method: "POST", }, ); if (data.success) { void this.fetchWorkflows(); } else { this.notify.toast({ message: msg("Something went wrong, couldn't cancel crawl."), variant: "danger", icon: "exclamation-octagon", id: "crawl-stop-error", }); } } } private async stop(crawlId: ListWorkflow["lastCrawlId"]) { if (!crawlId) return; if (window.confirm(msg("Are you sure you want to stop the crawl?"))) { const data = await this.api.fetch<{ success: boolean }>( `/orgs/${this.orgId}/crawls/${crawlId}/stop`, { method: "POST", }, ); if (data.success) { void this.fetchWorkflows(); } else { this.notify.toast({ message: msg("Something went wrong, couldn't stop crawl."), variant: "danger", icon: "exclamation-octagon", id: "crawl-stop-error", }); } } } private async runNow(workflow: ListWorkflow): Promise { try { await this.api.fetch( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}/run`, { method: "POST", }, ); this.notify.toast({ message: msg( html`Started crawl from ${this.renderName(workflow)}.
Watch crawl`, ), variant: "success", icon: "check2-circle", duration: 8000, }); await this.fetchWorkflows(); // Scroll to top of list this.scrollIntoView({ behavior: "smooth" }); } catch (e) { let message = msg("Sorry, couldn't run crawl at this time."); if (isApiError(e) && e.statusCode === 403) { if (e.details === "storage_quota_reached") { message = msg("Your org does not have enough storage to run crawls."); } else if (e.details === "exec_minutes_quota_reached") { message = msg( "Your org has used all of its execution minutes for this month.", ); } else { message = msg("You do not have permission to run crawls."); } } else if (isApiError(e) && e.details == "proxy_not_found") { message = msg( "Your org doesn't have permission to use the proxy configured for this crawl.", ); } this.notify.toast({ message: message, variant: "danger", icon: "exclamation-octagon", id: "crawl-start-error", }); } } private async fetchConfigSearchValues() { try { const data: { crawlIds: string[]; names: string[]; descriptions: string[]; firstSeeds: string[]; } = await this.api.fetch( `/orgs/${this.orgId}/crawlconfigs/search-values`, ); // Update search/filter collection const toSearchItem = (key: SearchFields) => (value: string) => ({ [key]: value, }); this.searchOptions = [ ...data.names.map(toSearchItem("name")), ...data.firstSeeds.map(toSearchItem("firstSeed")), ]; } catch (e) { console.debug(e); } } private async getWorkflow(workflow: ListWorkflow): Promise { const data: Workflow = await this.api.fetch( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}`, ); return data; } private async getSeeds(workflow: ListWorkflow) { // NOTE Returns first 1000 seeds (backend pagination max) const data = await this.api.fetch>( `/orgs/${this.orgId}/crawlconfigs/${workflow.id}/seeds`, ); return data; } private async getSeedFile(seedFileId: string) { const data = await this.api.fetch( `/orgs/${this.orgId}/files/${seedFileId}`, ); return data; } }