Run and delete crawl templates from list view (#94)
This commit is contained in:
sua yoo
GitHub
parent
b506442b21
commit
9ed216ba05
@ -1,9 +1,26 @@
|
||||
import { state, property } from "lit/decorators.js";
|
||||
import { msg, localized, str } from "@lit/localize";
|
||||
import cronParser from "cron-parser";
|
||||
|
||||
import type { AuthState } from "../../utils/AuthService";
|
||||
import LiteElement, { html } from "../../utils/LiteElement";
|
||||
import type { CrawlTemplate } from "./types";
|
||||
import type { CrawlConfig } from "./types";
|
||||
|
||||
type CrawlTemplate = {
|
||||
id: string;
|
||||
name: string;
|
||||
schedule: string;
|
||||
user: string;
|
||||
crawlCount: number;
|
||||
lastCrawlId: string;
|
||||
lastCrawlTime: string;
|
||||
currCrawlId: string;
|
||||
config: CrawlConfig;
|
||||
};
|
||||
type RunningCrawlsMap = {
|
||||
/** Map of configId: crawlId */
|
||||
[configId: string]: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Usage:
|
||||
@ -19,31 +36,289 @@ export class CrawlTemplatesList extends LiteElement {
|
||||
@property({ type: String })
|
||||
archiveId!: string;
|
||||
|
||||
@property({ type: Array })
|
||||
@state()
|
||||
crawlTemplates?: CrawlTemplate[];
|
||||
|
||||
@state()
|
||||
private serverError?: string;
|
||||
runningCrawlsMap: RunningCrawlsMap = {};
|
||||
|
||||
private get timeZone() {
|
||||
return Intl.DateTimeFormat().resolvedOptions().timeZone;
|
||||
}
|
||||
|
||||
async firstUpdated() {
|
||||
try {
|
||||
this.crawlTemplates = await this.getCrawlTemplates();
|
||||
if (!this.crawlTemplates.length) {
|
||||
this.navTo(`/archives/${this.archiveId}/crawl-templates/new`);
|
||||
}
|
||||
} catch (e) {
|
||||
this.notify({
|
||||
message: msg("Sorry, couldn't retrieve crawl templates at this time."),
|
||||
type: "danger",
|
||||
icon: "exclamation-octagon",
|
||||
duration: 10000,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
render() {
|
||||
if (!this.crawlTemplates) {
|
||||
return html`<div
|
||||
class="w-full flex items-center justify-center my-24 text-4xl"
|
||||
>
|
||||
<sl-spinner></sl-spinner>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
return html`
|
||||
<div class="text-center">
|
||||
<sl-button
|
||||
<div class="text-center"></div>
|
||||
|
||||
<div
|
||||
class=${this.crawlTemplates.length
|
||||
? "grid sm:grid-cols-2 md:grid-cols-3 gap-4 mb-4"
|
||||
: "flex justify-center"}
|
||||
>
|
||||
<div
|
||||
class="col-span-1 bg-slate-50 border border-dotted border-primary text-primary rounded px-6 py-4 flex items-center justify-center"
|
||||
@click=${() =>
|
||||
this.navTo(`/archives/${this.archiveId}/crawl-templates/new`)}
|
||||
role="button"
|
||||
>
|
||||
<sl-icon slot="prefix" name="plus-square-dotted"></sl-icon>
|
||||
${msg("Create new crawl template")}
|
||||
</sl-button>
|
||||
<sl-icon class="mr-2" name="plus-square-dotted"></sl-icon>
|
||||
<span
|
||||
class="mr-2 ${this.crawlTemplates.length
|
||||
? "text-sm"
|
||||
: "font-medium"}"
|
||||
>${msg("Create New Crawl Template")}</span
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
${this.crawlTemplates?.map(
|
||||
(template) => html`<div>${template.id}</div>`
|
||||
<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-4">
|
||||
${this.crawlTemplates.map(
|
||||
(t) =>
|
||||
html`<div
|
||||
class="col-span-1 p-1 border hover:border-indigo-200 rounded text-sm transition-colors"
|
||||
aria-label=${t.name}
|
||||
>
|
||||
<header class="flex">
|
||||
<a
|
||||
href=${`/archives/${this.archiveId}/crawl-templates/${t.id}`}
|
||||
class="block flex-1 px-3 pt-3 font-medium hover:underline whitespace-nowrap truncate mb-1"
|
||||
title=${t.name}
|
||||
@click=${this.navLink}
|
||||
>
|
||||
${t.name || "?"}
|
||||
</a>
|
||||
|
||||
<sl-dropdown>
|
||||
<sl-icon-button
|
||||
slot="trigger"
|
||||
name="three-dots-vertical"
|
||||
label="More"
|
||||
style="font-size: 1rem"
|
||||
></sl-icon-button>
|
||||
|
||||
<ul role="menu">
|
||||
<li
|
||||
class="px-4 py-2 text-danger hover:bg-danger hover:text-white cursor-pointer"
|
||||
role="menuitem"
|
||||
@click=${(e: any) => {
|
||||
// Close dropdown before deleting template
|
||||
e.target.closest("sl-dropdown").hide();
|
||||
|
||||
this.deleteTemplate(t);
|
||||
}}
|
||||
>
|
||||
${msg("Delete")}
|
||||
</li>
|
||||
</ul>
|
||||
</sl-dropdown>
|
||||
</header>
|
||||
|
||||
<div class="px-3 pb-3 flex justify-between items-end">
|
||||
<div class="grid gap-1 text-xs">
|
||||
<div
|
||||
class="font-mono whitespace-nowrap truncate text-gray-500"
|
||||
title=${t.config.seeds.join(", ")}
|
||||
>
|
||||
${t.config.seeds.join(", ")}
|
||||
</div>
|
||||
<div class="font-mono text-purple-500">
|
||||
${t.crawlCount === 1
|
||||
? msg(str`${t.crawlCount} crawl`)
|
||||
: msg(
|
||||
str`${(t.crawlCount || 0).toLocaleString()} crawls`
|
||||
)}
|
||||
</div>
|
||||
<div class="text-gray-500">
|
||||
${msg(html`Last:
|
||||
<span
|
||||
><sl-format-date
|
||||
date=${t.lastCrawlTime}
|
||||
month="2-digit"
|
||||
day="2-digit"
|
||||
year="2-digit"
|
||||
hour="numeric"
|
||||
minute="numeric"
|
||||
time-zone=${this.timeZone}
|
||||
></sl-format-date
|
||||
></span>`)}
|
||||
</div>
|
||||
<div class="text-gray-500">
|
||||
${t.schedule
|
||||
? msg(html`Next:
|
||||
<sl-format-date
|
||||
date="${cronParser
|
||||
.parseExpression(t.schedule, {
|
||||
utc: true,
|
||||
})
|
||||
.next()
|
||||
.toString()}"
|
||||
month="2-digit"
|
||||
day="2-digit"
|
||||
year="2-digit"
|
||||
hour="numeric"
|
||||
minute="numeric"
|
||||
time-zone=${this.timeZone}
|
||||
></sl-format-date>`)
|
||||
: html`<span class="text-gray-400"
|
||||
>${msg("No schedule")}</span
|
||||
>`}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<button
|
||||
class="text-xs border rounded-sm px-2 h-7 ${this
|
||||
.runningCrawlsMap[t.id]
|
||||
? "bg-purple-50"
|
||||
: "bg-white"} border-purple-200 hover:border-purple-500 text-purple-600 transition-colors"
|
||||
@click=${() =>
|
||||
this.runningCrawlsMap[t.id]
|
||||
? this.navTo(
|
||||
`/archives/${this.archiveId}/crawls/${
|
||||
this.runningCrawlsMap[t.id]
|
||||
}`
|
||||
)
|
||||
: this.runNow(t)}
|
||||
>
|
||||
<span>
|
||||
${this.runningCrawlsMap[t.id]
|
||||
? msg("View crawl")
|
||||
: msg("Run now")}
|
||||
</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>`
|
||||
)}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch crawl templates and record running crawls
|
||||
* associated with the crawl templates
|
||||
**/
|
||||
private async getCrawlTemplates(): Promise<CrawlTemplate[]> {
|
||||
type CrawlConfig = Omit<CrawlTemplate, "config"> & {
|
||||
config: Omit<CrawlTemplate["config"], "seeds"> & {
|
||||
seeds: (string | { url: string })[];
|
||||
};
|
||||
};
|
||||
|
||||
const data: { crawlConfigs: CrawlConfig[] } = await this.apiFetch(
|
||||
`/archives/${this.archiveId}/crawlconfigs`,
|
||||
this.authState!
|
||||
);
|
||||
|
||||
const crawlConfigs: CrawlTemplate[] = [];
|
||||
const runningCrawlsMap: RunningCrawlsMap = {};
|
||||
|
||||
data.crawlConfigs.forEach(({ config, ...configMeta }) => {
|
||||
crawlConfigs.push({
|
||||
...configMeta,
|
||||
config: {
|
||||
...config,
|
||||
// Flatten seeds into array of URL strings
|
||||
seeds: config.seeds.map((seed) =>
|
||||
typeof seed === "string" ? seed : seed.url
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
if (configMeta.currCrawlId) {
|
||||
runningCrawlsMap[configMeta.id] = configMeta.currCrawlId;
|
||||
}
|
||||
});
|
||||
|
||||
this.runningCrawlsMap = runningCrawlsMap;
|
||||
|
||||
return crawlConfigs;
|
||||
}
|
||||
|
||||
private async deleteTemplate(template: CrawlTemplate): Promise<void> {
|
||||
try {
|
||||
await this.apiFetch(
|
||||
`/archives/${this.archiveId}/crawlconfigs/${template.id}`,
|
||||
this.authState!,
|
||||
{
|
||||
method: "DELETE",
|
||||
}
|
||||
);
|
||||
|
||||
this.notify({
|
||||
message: msg(str`Deleted <strong>${template.name}</strong>.`),
|
||||
type: "success",
|
||||
icon: "check2-circle",
|
||||
});
|
||||
|
||||
this.crawlTemplates = this.crawlTemplates!.filter(
|
||||
(t) => t.id !== template.id
|
||||
);
|
||||
} catch {
|
||||
this.notify({
|
||||
message: msg("Sorry, couldn't delete crawl template at this time."),
|
||||
type: "danger",
|
||||
icon: "exclamation-octagon",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async runNow(template: CrawlTemplate): Promise<void> {
|
||||
try {
|
||||
const data = await this.apiFetch(
|
||||
`/archives/${this.archiveId}/crawlconfigs/${template.id}/run`,
|
||||
this.authState!,
|
||||
{
|
||||
method: "POST",
|
||||
}
|
||||
);
|
||||
|
||||
const crawlId = data.started;
|
||||
|
||||
this.runningCrawlsMap = {
|
||||
...this.runningCrawlsMap,
|
||||
[template.id]: crawlId,
|
||||
};
|
||||
|
||||
this.notify({
|
||||
message: msg(
|
||||
str`Started crawl from <strong>${template.name}</strong>. <br /><a class="underline hover:no-underline" href="/archives/${this.archiveId}/crawls/${data.run_now_job}">View crawl</a>`
|
||||
),
|
||||
type: "success",
|
||||
icon: "check2-circle",
|
||||
duration: 10000,
|
||||
});
|
||||
} catch {
|
||||
this.notify({
|
||||
message: msg("Sorry, couldn't run crawl at this time."),
|
||||
type: "danger",
|
||||
icon: "exclamation-octagon",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
customElements.define("btrix-crawl-templates-list", CrawlTemplatesList);
|
||||
|
||||
@ -5,7 +5,16 @@ import cronParser from "cron-parser";
|
||||
import type { AuthState } from "../../utils/AuthService";
|
||||
import LiteElement, { html } from "../../utils/LiteElement";
|
||||
import { getLocaleTimeZone } from "../../utils/localization";
|
||||
import type { CrawlTemplate } from "./types";
|
||||
import type { CrawlConfig } from "./types";
|
||||
|
||||
export type NewCrawlTemplate = {
|
||||
id?: string;
|
||||
name: string;
|
||||
schedule: string;
|
||||
runNow: boolean;
|
||||
crawlTimeout?: number;
|
||||
config: CrawlConfig;
|
||||
};
|
||||
|
||||
const initialValues = {
|
||||
name: "",
|
||||
@ -448,7 +457,7 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
const crawlTimeoutMinutes = formData.get("crawlTimeoutMinutes");
|
||||
const pageLimit = formData.get("limit");
|
||||
const seedUrlsStr = formData.get("seedUrls");
|
||||
const template: Partial<CrawlTemplate> = {
|
||||
const template: Partial<NewCrawlTemplate> = {
|
||||
name: formData.get("name") as string,
|
||||
schedule: this.getUTCSchedule(),
|
||||
runNow: this.isRunNow,
|
||||
|
||||
@ -7,7 +7,6 @@ import type { ArchiveData } from "../../utils/archives";
|
||||
import LiteElement, { html } from "../../utils/LiteElement";
|
||||
import { needLogin } from "../../utils/auth";
|
||||
import { isOwner } from "../../utils/archives";
|
||||
import type { CrawlTemplate } from "./types";
|
||||
import "./crawl-templates-list";
|
||||
import "./crawl-templates-new";
|
||||
|
||||
@ -40,9 +39,6 @@ export class Archive extends LiteElement {
|
||||
@state()
|
||||
private archive?: ArchiveData;
|
||||
|
||||
@state()
|
||||
private crawlTemplates?: CrawlTemplate[];
|
||||
|
||||
@state()
|
||||
private successfullyInvitedEmail?: string;
|
||||
|
||||
@ -61,17 +57,7 @@ export class Archive extends LiteElement {
|
||||
}
|
||||
|
||||
async updated(changedProperties: any) {
|
||||
if (
|
||||
changedProperties.has("archiveTab") &&
|
||||
this.archiveTab === "crawl-templates" &&
|
||||
!this.isNewResourceTab
|
||||
) {
|
||||
this.crawlTemplates = await this.getCrawlTemplates();
|
||||
|
||||
if (!this.crawlTemplates.length) {
|
||||
this.navTo(`/archives/${this.archiveId}/crawl-templates/new`);
|
||||
}
|
||||
} else if (changedProperties.has("isAddingMember") && this.isAddingMember) {
|
||||
if (changedProperties.has("isAddingMember") && this.isAddingMember) {
|
||||
this.successfullyInvitedEmail = undefined;
|
||||
}
|
||||
}
|
||||
@ -178,7 +164,6 @@ export class Archive extends LiteElement {
|
||||
return html`<btrix-crawl-templates-list
|
||||
.authState=${this.authState!}
|
||||
.archiveId=${this.archiveId!}
|
||||
.crawlTemplates=${this.crawlTemplates}
|
||||
></btrix-crawl-templates-list>`;
|
||||
}
|
||||
|
||||
@ -267,15 +252,6 @@ export class Archive extends LiteElement {
|
||||
return data;
|
||||
}
|
||||
|
||||
async getCrawlTemplates(): Promise<CrawlTemplate[]> {
|
||||
const data = await this.apiFetch(
|
||||
`/archives/${this.archiveId}/crawlconfigs`,
|
||||
this.authState!
|
||||
);
|
||||
|
||||
return data.crawl_configs;
|
||||
}
|
||||
|
||||
onInviteSuccess(
|
||||
event: CustomEvent<{ inviteEmail: string; isExistingUser: boolean }>
|
||||
) {
|
||||
|
||||
@ -1,12 +1,5 @@
|
||||
export type CrawlTemplate = {
|
||||
id?: string;
|
||||
name: string;
|
||||
schedule: string;
|
||||
runNow: boolean;
|
||||
crawlTimeout?: number;
|
||||
config: {
|
||||
seeds: string[];
|
||||
scopeType?: string;
|
||||
limit?: number;
|
||||
};
|
||||
export type CrawlConfig = {
|
||||
seeds: string[];
|
||||
scopeType?: string;
|
||||
limit?: number;
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user