Show only running crawls in superadmin view (#1015)
- Show separate crawls list for admin view, fixes #1010
This commit is contained in:
sua yoo
GitHub
parent
6506965d98
commit
7069b33646
@ -1,13 +1,41 @@
|
||||
import { state, property } from "lit/decorators.js";
|
||||
import { when } from "lit/directives/when.js";
|
||||
import { msg, localized, str } from "@lit/localize";
|
||||
import type { SlSelect } from "@shoelace-style/shoelace";
|
||||
import queryString from "query-string";
|
||||
|
||||
import type { PageChangeEvent } from "../components/pagination";
|
||||
import { CrawlStatus } from "../components/crawl-status";
|
||||
import type { AuthState } from "../utils/AuthService";
|
||||
import LiteElement, { html } from "../utils/LiteElement";
|
||||
import { needLogin } from "../utils/auth";
|
||||
import type { Crawl } from "../types/crawler";
|
||||
import { ROUTES } from "../routes";
|
||||
import { activeCrawlStates, inactiveCrawlStates } from "../utils/crawler";
|
||||
import type { Crawl, CrawlState } from "../types/crawler";
|
||||
import type { APIPaginationQuery, APIPaginatedList } from "../types/api";
|
||||
import "./org/workflow-detail";
|
||||
import "./org/crawls-list";
|
||||
import { PropertyValueMap } from "lit";
|
||||
|
||||
type SortField = "started" | "firstSeed" | "fileSize";
|
||||
type SortDirection = "asc" | "desc";
|
||||
const sortableFields: Record<
|
||||
SortField,
|
||||
{ label: string; defaultDirection?: SortDirection }
|
||||
> = {
|
||||
started: {
|
||||
label: msg("Date Started"),
|
||||
defaultDirection: "desc",
|
||||
},
|
||||
firstSeed: {
|
||||
label: msg("Crawl Start URL"),
|
||||
defaultDirection: "desc",
|
||||
},
|
||||
fileSize: {
|
||||
label: msg("File Size"),
|
||||
defaultDirection: "desc",
|
||||
},
|
||||
};
|
||||
const ABORT_REASON_THROTTLE = "throttled";
|
||||
|
||||
@needLogin
|
||||
@localized()
|
||||
@ -21,17 +49,49 @@ export class Crawls extends LiteElement {
|
||||
@state()
|
||||
private crawl?: Crawl;
|
||||
|
||||
willUpdate(changedProperties: Map<string, any>) {
|
||||
@state()
|
||||
private crawls?: APIPaginatedList;
|
||||
|
||||
@state()
|
||||
private orderBy: {
|
||||
field: SortField;
|
||||
direction: SortDirection;
|
||||
} = {
|
||||
field: "started",
|
||||
direction: sortableFields["started"].defaultDirection!,
|
||||
};
|
||||
|
||||
@state()
|
||||
private filterBy: Partial<Record<keyof Crawl, any>> = {
|
||||
state: activeCrawlStates,
|
||||
};
|
||||
|
||||
// Use to cancel requests
|
||||
private getCrawlsController: AbortController | null = null;
|
||||
|
||||
protected willUpdate(changedProperties: Map<string, any>) {
|
||||
if (changedProperties.has("crawlId") && this.crawlId) {
|
||||
this.fetchWorkflowId();
|
||||
} else {
|
||||
if (
|
||||
changedProperties.has("filterBy") ||
|
||||
changedProperties.has("orderBy")
|
||||
) {
|
||||
this.fetchCrawls();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
disconnectedCallback(): void {
|
||||
this.cancelInProgressGetCrawls();
|
||||
super.disconnectedCallback();
|
||||
}
|
||||
|
||||
render() {
|
||||
return html` <div
|
||||
class="w-full max-w-screen-lg mx-auto px-3 py-4 box-border"
|
||||
>
|
||||
${this.crawlId ? this.renderDetail() : this.renderList()}
|
||||
${this.crawlId ? this.renderDetail() : this.renderCrawls()}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
@ -49,18 +109,194 @@ export class Crawls extends LiteElement {
|
||||
`;
|
||||
}
|
||||
|
||||
private renderList() {
|
||||
return html`<btrix-crawls-list
|
||||
.authState=${this.authState}
|
||||
crawlsBaseUrl=${ROUTES.crawls}
|
||||
crawlsAPIBaseUrl="/orgs/all/crawls"
|
||||
artifactType="crawl"
|
||||
isCrawler
|
||||
isAdminView
|
||||
shouldFetch
|
||||
></btrix-crawls-list>`;
|
||||
private renderCrawls() {
|
||||
return html`
|
||||
<main>
|
||||
<header class="contents">
|
||||
<div class="flex justify-between w-full pb-4 mb-3 border-b">
|
||||
<h1 class="text-xl font-semibold h-8">
|
||||
${msg("All Running Crawls")}
|
||||
</h1>
|
||||
</div>
|
||||
<div
|
||||
class="sticky z-10 mb-3 top-2 p-4 bg-neutral-50 border rounded-lg"
|
||||
>
|
||||
${this.renderControls()}
|
||||
</div>
|
||||
</header>
|
||||
|
||||
${when(
|
||||
this.crawls,
|
||||
() => {
|
||||
const { items, page, total, pageSize } = this.crawls!;
|
||||
const hasCrawlItems = items.length;
|
||||
return html`
|
||||
<section>
|
||||
${hasCrawlItems
|
||||
? this.renderCrawlList()
|
||||
: this.renderEmptyState()}
|
||||
</section>
|
||||
${when(
|
||||
hasCrawlItems || page > 1,
|
||||
() => html`
|
||||
<footer class="mt-6 flex justify-center">
|
||||
<btrix-pagination
|
||||
page=${page}
|
||||
totalCount=${total}
|
||||
size=${pageSize}
|
||||
@page-change=${async (e: PageChangeEvent) => {
|
||||
await this.fetchCrawls({
|
||||
page: e.detail.page,
|
||||
});
|
||||
|
||||
// Scroll to top of list
|
||||
// TODO once deep-linking is implemented, scroll to top of pushstate
|
||||
this.scrollIntoView({ behavior: "smooth" });
|
||||
}}
|
||||
></btrix-pagination>
|
||||
</footer>
|
||||
`
|
||||
)}
|
||||
`;
|
||||
},
|
||||
() => html`
|
||||
<div class="w-full flex items-center justify-center my-12 text-2xl">
|
||||
<sl-spinner></sl-spinner>
|
||||
</div>
|
||||
`
|
||||
)}
|
||||
</main>
|
||||
`;
|
||||
}
|
||||
|
||||
private renderControls() {
|
||||
const viewPlaceholder = msg("Any Active Status");
|
||||
const viewOptions = activeCrawlStates;
|
||||
return html`
|
||||
<div class="flex gap-2 items-center justify-end">
|
||||
<div class="flex items-center">
|
||||
<div class="text-neutral-500 mx-2">${msg("View:")}</div>
|
||||
<sl-select
|
||||
id="stateSelect"
|
||||
class="flex-1 md:w-[14.5rem]"
|
||||
size="small"
|
||||
pill
|
||||
multiple
|
||||
max-options-visible="1"
|
||||
placeholder=${viewPlaceholder}
|
||||
@sl-change=${async (e: CustomEvent) => {
|
||||
const value = (e.target as SlSelect).value as CrawlState[];
|
||||
await this.updateComplete;
|
||||
this.filterBy = {
|
||||
...this.filterBy,
|
||||
state: value,
|
||||
};
|
||||
}}
|
||||
>
|
||||
${viewOptions.map(this.renderStatusMenuItem)}
|
||||
</sl-select>
|
||||
</div>
|
||||
|
||||
<div class="flex items-center">
|
||||
<div class="whitespace-nowrap text-neutral-500 mx-2">
|
||||
${msg("Sort by:")}
|
||||
</div>
|
||||
<div class="grow flex">${this.renderSortControl()}</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
private renderSortControl() {
|
||||
const options = Object.entries(sortableFields).map(
|
||||
([value, { label }]) => html`
|
||||
<sl-option value=${value}>${label}</sl-option>
|
||||
`
|
||||
);
|
||||
return html`
|
||||
<sl-select
|
||||
class="flex-1 md:w-[10rem]"
|
||||
size="small"
|
||||
pill
|
||||
value=${this.orderBy.field}
|
||||
@sl-change=${(e: Event) => {
|
||||
const field = (e.target as HTMLSelectElement).value as SortField;
|
||||
this.orderBy = {
|
||||
field: field,
|
||||
direction:
|
||||
sortableFields[field].defaultDirection || this.orderBy.direction,
|
||||
};
|
||||
}}
|
||||
>
|
||||
${options}
|
||||
</sl-select>
|
||||
<sl-icon-button
|
||||
name="arrow-down-up"
|
||||
label=${msg("Reverse sort")}
|
||||
@click=${() => {
|
||||
this.orderBy = {
|
||||
...this.orderBy,
|
||||
direction: this.orderBy.direction === "asc" ? "desc" : "asc",
|
||||
};
|
||||
}}
|
||||
></sl-icon-button>
|
||||
`;
|
||||
}
|
||||
|
||||
private renderStatusMenuItem = (state: CrawlState) => {
|
||||
const { icon, label } = CrawlStatus.getContent(state);
|
||||
|
||||
return html`<sl-option value=${state}>${icon}${label}</sl-option>`;
|
||||
};
|
||||
|
||||
private renderCrawlList() {
|
||||
if (!this.crawls) return;
|
||||
|
||||
return html`
|
||||
<btrix-crawl-list baseUrl=${"/crawls/crawl"} artifactType="crawl">
|
||||
${this.crawls.items.map(this.renderCrawlItem)}
|
||||
</btrix-crawl-list>
|
||||
`;
|
||||
}
|
||||
|
||||
private renderEmptyState() {
|
||||
if (this.crawls?.page && this.crawls?.page > 1) {
|
||||
return html`
|
||||
<div class="border-t border-b py-5">
|
||||
<p class="text-center text-neutral-500">
|
||||
${msg("Could not find page.")}
|
||||
</p>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
return html`
|
||||
<div class="border-t border-b py-5">
|
||||
<p class="text-center text-neutral-500">
|
||||
${msg("No matching crawls found.")}
|
||||
</p>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
private renderCrawlItem = (crawl: Crawl) =>
|
||||
html`
|
||||
<btrix-crawl-list-item .crawl=${crawl}>
|
||||
<sl-menu slot="menu">
|
||||
<sl-menu-item
|
||||
@click=${() =>
|
||||
this.navTo(
|
||||
`/orgs/${crawl.oid}/artifacts/${
|
||||
crawl.type === "upload" ? "upload" : "crawl"
|
||||
}/${crawl.id}`
|
||||
)}
|
||||
>
|
||||
${msg("View Crawl Details")}
|
||||
</sl-menu-item>
|
||||
</sl-menu>
|
||||
</btrix-crawl-list-item>
|
||||
`;
|
||||
|
||||
private async fetchWorkflowId() {
|
||||
try {
|
||||
this.crawl = await this.getCrawl();
|
||||
@ -69,6 +305,63 @@ export class Crawls extends LiteElement {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch crawls and update internal state
|
||||
*/
|
||||
private async fetchCrawls(params?: APIPaginationQuery): Promise<void> {
|
||||
this.cancelInProgressGetCrawls();
|
||||
try {
|
||||
this.crawls = await this.getCrawls(params);
|
||||
} catch (e: any) {
|
||||
if (e === ABORT_REASON_THROTTLE) {
|
||||
console.debug("Fetch crawls aborted to throttle");
|
||||
} else {
|
||||
this.notify({
|
||||
message: msg("Sorry, couldn't retrieve crawls at this time."),
|
||||
variant: "danger",
|
||||
icon: "exclamation-octagon",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private cancelInProgressGetCrawls() {
|
||||
if (this.getCrawlsController) {
|
||||
this.getCrawlsController.abort(ABORT_REASON_THROTTLE);
|
||||
this.getCrawlsController = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async getCrawls(
|
||||
queryParams?: APIPaginationQuery & { state?: CrawlState[] }
|
||||
): Promise<APIPaginatedList> {
|
||||
const query = queryString.stringify(
|
||||
{
|
||||
...this.filterBy,
|
||||
...queryParams,
|
||||
page: queryParams?.page || this.crawls?.page || 1,
|
||||
pageSize: queryParams?.pageSize || this.crawls?.pageSize || 100,
|
||||
sortBy: this.orderBy.field,
|
||||
sortDirection: this.orderBy.direction === "desc" ? -1 : 1,
|
||||
},
|
||||
{
|
||||
arrayFormat: "comma",
|
||||
}
|
||||
);
|
||||
|
||||
this.getCrawlsController = new AbortController();
|
||||
const data = await this.apiFetch(
|
||||
`/orgs/all/crawls?${query}`,
|
||||
this.authState!,
|
||||
{
|
||||
signal: this.getCrawlsController.signal,
|
||||
}
|
||||
);
|
||||
this.getCrawlsController = null;
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private async getCrawl(): Promise<Crawl> {
|
||||
const data: Crawl = await this.apiFetch(
|
||||
`/orgs/all/crawls/${this.crawlId}/replay.json`,
|
||||
|
||||
@ -91,11 +91,6 @@ export class CrawlsList extends LiteElement {
|
||||
@property({ type: Boolean })
|
||||
isCrawler!: boolean;
|
||||
|
||||
// TODO better handling of using same crawls-list
|
||||
// component between superadmin view and regular view
|
||||
@property({ type: Boolean })
|
||||
isAdminView = false;
|
||||
|
||||
// e.g. `/org/${this.orgId}/crawls`
|
||||
@property({ type: String })
|
||||
crawlsBaseUrl!: string;
|
||||
@ -178,12 +173,6 @@ export class CrawlsList extends LiteElement {
|
||||
}
|
||||
|
||||
protected willUpdate(changedProperties: Map<string, any>) {
|
||||
if (changedProperties.has("isAdminView") && this.isAdminView === true) {
|
||||
this.orderBy = {
|
||||
field: "started",
|
||||
direction: sortableFields["started"].defaultDirection!,
|
||||
};
|
||||
}
|
||||
if (
|
||||
changedProperties.has("shouldFetch") ||
|
||||
changedProperties.get("crawlsBaseUrl") ||
|
||||
@ -226,10 +215,7 @@ export class CrawlsList extends LiteElement {
|
||||
changedProperties.has("crawlsBaseUrl") ||
|
||||
changedProperties.has("crawlsAPIBaseUrl")
|
||||
) {
|
||||
// TODO add back when API supports `orgs/all/crawlconfigs`
|
||||
if (!this.isAdminView) {
|
||||
this.fetchConfigSearchValues();
|
||||
}
|
||||
this.fetchConfigSearchValues();
|
||||
}
|
||||
}
|
||||
|
||||
@ -244,30 +230,21 @@ export class CrawlsList extends LiteElement {
|
||||
label: string;
|
||||
icon?: string;
|
||||
}[] = [
|
||||
{
|
||||
artifactType: null,
|
||||
label: msg("All"),
|
||||
},
|
||||
{
|
||||
artifactType: "crawl",
|
||||
icon: "gear-wide-connected",
|
||||
label: msg("Crawls"),
|
||||
},
|
||||
];
|
||||
|
||||
if (this.isAdminView) {
|
||||
listTypes.unshift({
|
||||
artifactType: "crawl",
|
||||
icon: "gear-wide",
|
||||
label: msg("Running Crawls"),
|
||||
});
|
||||
} else {
|
||||
listTypes.unshift({
|
||||
artifactType: null,
|
||||
label: msg("All"),
|
||||
});
|
||||
listTypes.push({
|
||||
{
|
||||
artifactType: "upload",
|
||||
icon: "upload",
|
||||
label: msg("Uploads"),
|
||||
});
|
||||
}
|
||||
},
|
||||
];
|
||||
|
||||
return html`
|
||||
<main>
|
||||
@ -376,25 +353,18 @@ export class CrawlsList extends LiteElement {
|
||||
}
|
||||
|
||||
private renderControls() {
|
||||
let viewPlaceholder = "";
|
||||
let viewOptions = [];
|
||||
if (this.isAdminView) {
|
||||
viewPlaceholder = msg("All Active Crawls");
|
||||
viewOptions = activeCrawlStates;
|
||||
} else {
|
||||
viewOptions = finishedCrawlStates;
|
||||
if (this.artifactType === "upload") {
|
||||
viewPlaceholder = msg("All Uploaded");
|
||||
} else {
|
||||
viewPlaceholder = msg("All Finished");
|
||||
}
|
||||
}
|
||||
const viewPlaceholder =
|
||||
this.artifactType === "upload"
|
||||
? msg("All Uploaded")
|
||||
: msg("All Finished");
|
||||
const viewOptions = finishedCrawlStates;
|
||||
|
||||
return html`
|
||||
<div
|
||||
class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-[minmax(0,100%)_fit-content(100%)_fit-content(100%)] gap-x-2 gap-y-2 items-center"
|
||||
>
|
||||
<div class="col-span-1 md:col-span-2 lg:col-span-1">
|
||||
${when(!this.isAdminView, () => this.renderSearch())}
|
||||
${this.renderSearch()}
|
||||
</div>
|
||||
<div class="flex items-center">
|
||||
<div class="text-neutral-500 mx-2">${msg("View:")}</div>
|
||||
@ -586,7 +556,7 @@ export class CrawlsList extends LiteElement {
|
||||
|
||||
return html`
|
||||
<btrix-crawl-list
|
||||
baseUrl=${this.isAdminView ? "/crawls/crawl" : ""}
|
||||
baseUrl=""
|
||||
artifactType=${ifDefined(this.artifactType || undefined)}
|
||||
>
|
||||
${this.crawls.items.map(this.renderCrawlItem)}
|
||||
@ -781,12 +751,6 @@ export class CrawlsList extends LiteElement {
|
||||
state: this.filterBy.state || finishedCrawlStates,
|
||||
});
|
||||
break;
|
||||
// case "crawl":
|
||||
// crawls = await this.getCrawls({
|
||||
// ...params,
|
||||
// state: this.filterBy.state || activeCrawlStates,
|
||||
// });
|
||||
// break;
|
||||
case "upload":
|
||||
crawls = await this.getUploads(params);
|
||||
break;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user