import type { HTMLTemplateResult } from "lit";
import { state, property } from "lit/decorators.js";
import { ifDefined } from "lit/directives/if-defined.js";
import { msg, localized, str } from "@lit/localize";
import cronstrue from "cronstrue"; // TODO localize
import { parse as yamlToJson, stringify as jsonToYaml } from "yaml";
import type { AuthState } from "../../utils/AuthService";
import LiteElement, { html } from "../../utils/LiteElement";
import type { CrawlTemplate, CrawlConfig } from "./types";
import { getUTCSchedule } from "./utils";
import "../../components/crawl-scheduler";
const SEED_URLS_MAX = 3;
/**
* Usage:
* ```ts
*
* ```
*/
@localized()
export class CrawlTemplatesDetail extends LiteElement {
@property({ type: Object })
authState!: AuthState;
@property({ type: String })
archiveId!: string;
@property({ type: String })
crawlConfigId!: string;
@state()
private crawlTemplate?: CrawlTemplate;
@state()
private showAllSeedURLs: boolean = false;
@state()
private isConfigCodeView: boolean = false;
/** YAML or stringified JSON config */
@state()
private configCode: string = "";
@state()
private isSubmittingUpdate: boolean = false;
@state()
private openDialogName?: "name" | "config" | "schedule" | "scale";
@state()
private isDialogVisible: boolean = false;
updated(changedProperties: any) {
if (changedProperties.has("crawlConfigId")) {
this.initializeCrawlTemplate();
}
}
private async initializeCrawlTemplate() {
try {
this.crawlTemplate = await this.getCrawlTemplate();
// Show JSON editor view if complex initial config is specified
// (e.g. cloning a template) since form UI doesn't support
// all available fields in the config
const isComplexConfig = this.crawlTemplate.config.seeds.some(
(seed: any) => typeof seed !== "string"
);
if (isComplexConfig) {
this.isConfigCodeView = true;
}
this.configCode = jsonToYaml(this.crawlTemplate.config);
} catch (e: any) {
this.notify({
message:
e.statusCode === 404
? msg("Crawl template not found.")
: msg("Sorry, couldn't retrieve crawl template at this time."),
type: "danger",
icon: "exclamation-octagon",
});
}
}
render() {
return html`
${msg("Back to Crawl Templates")}
${this.renderInactiveNotice()}
${this.crawlTemplate?.name
? html`
${this.crawlTemplate.name}
${this.crawlTemplate.inactive
? ""
: html`
(this.openDialogName = "name")}
>
${msg("Edit")}
`}
`
: html` `}
${msg("Crawl Template")}
${this.crawlTemplate?.id}
${this.renderMenu()}
${this.renderCurrentlyRunningNotice()}
${msg("Configuration")}
${this.crawlTemplate?.oldId
? html`
`
: ""}
${this.crawlTemplate?.newId
? html`
`
: ""}
${this.renderConfiguration()}
${this.crawlTemplate?.inactive || !this.crawlTemplate
? ""
: html`
(this.openDialogName = "config")}
>
${msg("Edit")}
`}
${msg("Schedule")}
${this.renderSchedule()}
${this.crawlTemplate?.inactive || !this.crawlTemplate
? ""
: html`
(this.openDialogName = "schedule")}
>
${msg("Edit")}
`}
${msg("Crawls")}
${this.renderCrawls()}
${this.renderDialogs()}
`;
}
private renderMenu() {
if (!this.crawlTemplate) return;
const closeDropdown = (e: any) => {
e.target.closest("sl-dropdown").hide();
};
const menuItems: HTMLTemplateResult[] = [
html`
this.duplicateConfig()}
>
${msg("Duplicate crawl config")}
`,
];
if (!this.crawlTemplate.inactive) {
menuItems.unshift(html`
{
closeDropdown(e);
this.runNow();
}}
>
${msg("Run now")}
{
closeDropdown(e);
this.openDialogName = "name";
}}
>
${msg("Change name")}
{
closeDropdown(e);
this.openDialogName = "config";
}}
>
${msg("Change crawl configuration")}
{
closeDropdown(e);
this.openDialogName = "schedule";
}}
>
${msg("Change schedule")}
{
closeDropdown(e);
this.openDialogName = "scale";
}}
>
${msg("Change scale")}
`);
}
if (this.crawlTemplate.crawlCount && !this.crawlTemplate.inactive) {
menuItems.push(html`
{
closeDropdown(e);
this.deactivateTemplate();
}}
>
${msg("Deactivate")}
`);
}
if (!this.crawlTemplate.crawlCount) {
menuItems.push(html`
{
this.deleteTemplate();
}}
>
${msg("Delete")}
`);
}
return html`
${msg("Actions")}
${menuItems.map((item: HTMLTemplateResult) => item)}
`;
}
private renderInactiveNotice() {
if (this.crawlTemplate?.inactive) {
if (this.crawlTemplate?.newId) {
return html`
${msg("This crawl template is inactive.")}
${msg("Go to newer version")}
`;
}
return html`
${msg("This crawl template is inactive.")}
`;
}
return "";
}
private renderCurrentlyRunningNotice() {
if (this.crawlTemplate?.currCrawlId) {
return html`
${msg("View currently running crawl")}
`;
}
return "";
}
private renderDetails() {
return html`
${msg("Created at")}
${this.crawlTemplate?.created
? html`
`
: html` `}
${msg("Created by")}
${this.crawlTemplate?.userName ||
this.crawlTemplate?.userid ||
html` `}
`;
}
private renderEditName() {
if (!this.crawlTemplate) return;
return html`
(this.openDialogName = undefined)}
>${msg("Cancel")}
${msg("Save Changes")}
`;
}
private renderConfiguration() {
const seeds = this.crawlTemplate?.config.seeds || [];
const configCodeYaml = jsonToYaml(this.crawlTemplate?.config || {});
return html`
${msg("Seed URL")}
${msg("Scope Type")}
${msg("Page Limit")}
${seeds.length > SEED_URLS_MAX
? html`
(this.showAllSeedURLs = !this.showAllSeedURLs)}
>
${this.showAllSeedURLs
? msg("Show less")
: msg(str`Show
${seeds.length - SEED_URLS_MAX}
more`)}
`
: ""}
${msg("Include External Links")}
${this.crawlTemplate?.config.extraHops ? msg("Yes") : msg("No")}
${msg("Advanced Configuration")}
`;
}
private renderEditConfiguration() {
if (!this.crawlTemplate) return;
const shouldReplacingTemplate = this.crawlTemplate.crawlCount > 0;
return html`
${shouldReplacingTemplate
? html`
${msg(
"Editing the crawl configuration will replace this crawl template with a new version. All other settings will be kept the same."
)}
`
: ""}
${this.isConfigCodeView
? msg("Custom Config")
: msg("Crawl Configuration")}
(this.isConfigCodeView = e.target.checked)}
>
${msg("Advanced Editor")}
${this.renderSeedsCodeEditor()}
${this.renderSeedsForm()}
(this.openDialogName = undefined)}
>${msg("Cancel")}
${msg("Save Changes")}
`;
}
private renderSchedule() {
return html`
${msg("Recurring crawls")}
${this.crawlTemplate
? html`
${this.crawlTemplate.schedule
? // TODO localize
// NOTE human-readable string is in UTC, limitation of library
// currently being used.
// https://github.com/bradymholt/cRonstrue/issues/94
html`${cronstrue.toString(this.crawlTemplate.schedule, {
verbose: true,
})}
(in UTC time zone) `
: html`${msg("None")} `}
`
: html` `}
`;
}
private renderEditSchedule() {
if (!this.crawlTemplate) return;
return html`
(this.openDialogName = undefined)}
@submit=${this.handleSubmitEditSchedule}
>
`;
}
private renderCrawls() {
return html`
${msg("# of Crawls")}
${(this.crawlTemplate?.crawlCount || 0).toLocaleString()}
${msg("Crawl Scale")}
${this.crawlTemplate?.scale}
${!this.crawlTemplate || this.crawlTemplate.inactive
? ""
: html`
(this.openDialogName = "scale")}
>
${msg("Edit")}
`}
${msg("Currently Running Crawl")}
${this.crawlTemplate
? html`
${this.crawlTemplate.currCrawlId
? html` ${msg("View crawl")} `
: this.crawlTemplate.inactive
? ""
: html`${msg("None")} this.runNow()}
>
${msg("Run now")}
`}
`
: html` `}
${msg("Latest Crawl")}
${this.crawlTemplate?.lastCrawlId
? html`
${this.crawlTemplate.lastCrawlState.replace(/_/g, " ")}
${msg("View crawl")}
`
: html`${msg("None")} `}
`;
}
private renderEditScale() {
if (!this.crawlTemplate) return;
return html`
${msg("Standard")}
${msg("Big (2x)")}
${msg("Bigger (3x)")}
(this.openDialogName = undefined)}
>${msg("Cancel")}
${msg("Save Changes")}
`;
}
/**
* Render dialog for edit forms
*
* `openDialogName` shows/hides a specific dialog, while `isDialogVisible`
* renders/prevents rendering a dialog's content unless the dialog is visible
* in order to reset the dialog content on close.
*/
private renderDialogs() {
const dialogWidth = "36rem";
const resetScroll = (e: any) => {
const dialogBody = e.target.shadowRoot.querySelector('[part="body"]');
if (dialogBody) {
dialogBody.scrollTop = 0;
}
};
return html`
(this.openDialogName = undefined)}
@sl-show=${() => (this.isDialogVisible = true)}
@sl-after-hide=${() => (this.isDialogVisible = false)}
>
${this.isDialogVisible ? this.renderEditName() : ""}
(this.openDialogName = undefined)}
@sl-show=${() => (this.isDialogVisible = true)}
@sl-after-hide=${() => (this.isDialogVisible = false)}
@sl-after-show=${resetScroll}
>
${this.isDialogVisible ? this.renderEditConfiguration() : ""}
(this.openDialogName = undefined)}
@sl-show=${() => (this.isDialogVisible = true)}
@sl-after-hide=${() => (this.isDialogVisible = false)}
>
${this.isDialogVisible ? this.renderEditSchedule() : ""}
(this.openDialogName = undefined)}
@sl-show=${() => (this.isDialogVisible = true)}
@sl-after-hide=${() => (this.isDialogVisible = false)}
>
${this.isDialogVisible ? this.renderEditScale() : ""}
`;
}
private renderSeedsForm() {
return html`
Page
Page SPA
Prefix
Host
Domain
Any
${msg("Include External Links (“one hop out”)")}
${msg("pages")}
`;
}
private renderSeedsCodeEditor() {
return html`
{
this.configCode = e.detail.value;
}}
>
`;
}
async getCrawlTemplate(): Promise {
const data: CrawlTemplate = await this.apiFetch(
`/archives/${this.archiveId}/crawlconfigs/${this.crawlConfigId}`,
this.authState!
);
return data;
}
/**
* Create a new template using existing template data
*/
private async duplicateConfig() {
if (!this.crawlTemplate) return;
const config: CrawlTemplate["config"] = {
...this.crawlTemplate.config,
};
this.navTo(`/archives/${this.archiveId}/crawl-templates/new`, {
crawlTemplate: {
name: msg(str`${this.crawlTemplate.name} Copy`),
config,
},
});
this.notify({
message: msg(str`Copied crawl configuration to new template.`),
type: "success",
icon: "check2-circle",
});
}
private async handleSubmitEditName(e: { detail: { formData: FormData } }) {
const { formData } = e.detail;
const name = formData.get("name") as string;
await this.updateTemplate({ name });
this.openDialogName = undefined;
}
private async handleSubmitEditScale(e: { detail: { formData: FormData } }) {
const { formData } = e.detail;
const scale = formData.get("scale") as string;
await this.updateTemplate({ scale: +scale });
this.openDialogName = undefined;
}
private async handleSubmitEditConfiguration(e: {
detail: { formData: FormData };
}) {
const { formData } = e.detail;
let config: CrawlConfig;
if (this.isConfigCodeView) {
if (!this.configCode) return;
config = yamlToJson(this.configCode) as CrawlConfig;
} else {
const pageLimit = formData.get("limit") as string;
const seedUrlsStr = formData.get("seedUrls") as string;
config = {
seeds: seedUrlsStr.trim().replace(/,/g, " ").split(/\s+/g),
scopeType: formData.get("scopeType") as string,
limit: pageLimit ? +pageLimit : 0,
extraHops: formData.get("extraHopsOne") ? 1 : 0,
};
}
if (config) {
await this.createRevisedTemplate(config);
}
this.openDialogName = undefined;
}
private async handleSubmitEditSchedule(e: {
detail: { formData: FormData };
}) {
const { formData } = e.detail;
const interval = formData.get("scheduleInterval");
let schedule = "";
if (interval) {
schedule = getUTCSchedule({
interval: formData.get("scheduleInterval") as any,
hour: formData.get("scheduleHour") as any,
minute: formData.get("scheduleMinute") as any,
period: formData.get("schedulePeriod") as any,
});
}
await this.updateTemplate({ schedule });
this.openDialogName = undefined;
}
private async deactivateTemplate(): Promise {
if (!this.crawlTemplate) return;
try {
await this.apiFetch(
`/archives/${this.archiveId}/crawlconfigs/${this.crawlTemplate.id}`,
this.authState!,
{
method: "DELETE",
}
);
this.notify({
message: msg(
html`Deactivated ${this.crawlTemplate.name} .`
),
type: "success",
icon: "check2-circle",
});
} catch {
this.notify({
message: msg("Sorry, couldn't deactivate crawl template at this time."),
type: "danger",
icon: "exclamation-octagon",
});
}
}
private async deleteTemplate(): Promise {
if (!this.crawlTemplate) return;
const isDeactivating = this.crawlTemplate.crawlCount > 0;
try {
await this.apiFetch(
`/archives/${this.archiveId}/crawlconfigs/${this.crawlTemplate.id}`,
this.authState!,
{
method: "DELETE",
}
);
this.navTo(`/archives/${this.archiveId}/crawl-templates`);
this.notify({
message: isDeactivating
? msg(html`Deactivated ${this.crawlTemplate.name} .`)
: msg(html`Deleted ${this.crawlTemplate.name} .`),
type: "success",
icon: "check2-circle",
});
} catch {
this.notify({
message: isDeactivating
? msg("Sorry, couldn't deactivate crawl template at this time.")
: msg("Sorry, couldn't delete crawl template at this time."),
type: "danger",
icon: "exclamation-octagon",
});
}
}
private async runNow(): Promise {
try {
const data = await this.apiFetch(
`/archives/${this.archiveId}/crawlconfigs/${
this.crawlTemplate!.id
}/run`,
this.authState!,
{
method: "POST",
}
);
const crawlId = data.started;
this.crawlTemplate = {
...this.crawlTemplate,
currCrawlId: crawlId,
} as CrawlTemplate;
this.notify({
message: msg(
html`Started crawl from ${this.crawlTemplate!.name} .
View crawl `
),
type: "success",
icon: "check2-circle",
duration: 8000,
});
} catch {
this.notify({
message: msg("Sorry, couldn't run crawl at this time."),
type: "danger",
icon: "exclamation-octagon",
});
}
}
/**
* Create new crawl template with revised crawl configuration
* @param config Crawl config object
*/
private async createRevisedTemplate(config: CrawlConfig) {
this.isSubmittingUpdate = true;
const params = {
oldId: this.crawlTemplate!.id,
name: this.crawlTemplate!.name,
schedule: this.crawlTemplate!.schedule,
// runNow: this.crawlTemplate!.runNow,
// crawlTimeout: this.crawlTemplate!.crawlTimeout,
config,
};
try {
const data = await this.apiFetch(
`/archives/${this.archiveId}/crawlconfigs/`,
this.authState!,
{
method: "POST",
body: JSON.stringify(params),
}
);
console.log(data);
this.navTo(
`/archives/${this.archiveId}/crawl-templates/config/${data.added}`
);
this.notify({
message: msg("Crawl template updated."),
type: "success",
icon: "check2-circle",
});
} catch (e: any) {
console.error(e);
this.notify({
message: msg("Something went wrong, couldn't update crawl template."),
type: "danger",
icon: "exclamation-octagon",
});
}
this.isSubmittingUpdate = false;
}
/**
* Update crawl template properties
* @param params Crawl template properties to update
*/
private async updateTemplate(params: Partial): Promise {
console.log(params);
this.isSubmittingUpdate = true;
try {
const data = await this.apiFetch(
`/archives/${this.archiveId}/crawlconfigs/${this.crawlTemplate!.id}`,
this.authState!,
{
method: "PATCH",
body: JSON.stringify(params),
}
);
if (data.success === true) {
this.crawlTemplate = {
...this.crawlTemplate!,
...params,
};
this.notify({
message: msg("Successfully saved changes."),
type: "success",
icon: "check2-circle",
});
} else {
throw data;
}
} catch (e: any) {
console.error(e);
this.notify({
message: msg("Something went wrong, couldn't update crawl template."),
type: "danger",
icon: "exclamation-octagon",
});
}
this.isSubmittingUpdate = false;
}
/**
* Stop propgation of sl-select events.
* Prevents bug where sl-dialog closes when dropdown closes
* https://github.com/shoelace-style/shoelace/issues/170
*/
private stopProp(e: CustomEvent) {
e.stopPropagation();
}
}
customElements.define("btrix-crawl-templates-detail", CrawlTemplatesDetail);