import type { HTMLTemplateResult } from "lit"; import { state, property } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { msg, localized, str } from "@lit/localize"; import cronstrue from "cronstrue"; // TODO localize import { parse as yamlToJson, stringify as jsonToYaml } from "yaml"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import type { CrawlTemplate, CrawlConfig } from "./types"; import { getUTCSchedule } from "./utils"; import "../../components/crawl-scheduler"; const SEED_URLS_MAX = 3; /** * Usage: * ```ts * * ``` */ @localized() export class CrawlTemplatesDetail extends LiteElement { @property({ type: Object }) authState!: AuthState; @property({ type: String }) archiveId!: string; @property({ type: String }) crawlConfigId!: string; @state() private crawlTemplate?: CrawlTemplate; @state() private showAllSeedURLs: boolean = false; @state() private isConfigCodeView: boolean = false; /** YAML or stringified JSON config */ @state() private configCode: string = ""; @state() private isSubmittingUpdate: boolean = false; @state() private openDialogName?: "name" | "config" | "schedule" | "scale"; @state() private isDialogVisible: boolean = false; updated(changedProperties: any) { if (changedProperties.has("crawlConfigId")) { this.initializeCrawlTemplate(); } } private async initializeCrawlTemplate() { try { this.crawlTemplate = await this.getCrawlTemplate(); // Show JSON editor view if complex initial config is specified // (e.g. cloning a template) since form UI doesn't support // all available fields in the config const isComplexConfig = this.crawlTemplate.config.seeds.some( (seed: any) => typeof seed !== "string" ); if (isComplexConfig) { this.isConfigCodeView = true; } this.configCode = jsonToYaml(this.crawlTemplate.config); } catch (e: any) { this.notify({ message: e.statusCode === 404 ? msg("Crawl template not found.") : msg("Sorry, couldn't retrieve crawl template at this time."), type: "danger", icon: "exclamation-octagon", }); } } render() { return html`
${this.renderInactiveNotice()}

${this.crawlTemplate?.name ? html` ${this.crawlTemplate.name} ${this.crawlTemplate.inactive ? "" : html` (this.openDialogName = "name")} > ${msg("Edit")} `} ` : html``}

${msg("Crawl Template")}
${this.crawlTemplate?.id}
${this.renderMenu()}
${this.renderDetails()}
${this.renderCurrentlyRunningNotice()}

${msg("Configuration")}

${this.crawlTemplate?.oldId ? html` ` : ""} ${this.crawlTemplate?.newId ? html` ` : ""}
${this.renderConfiguration()}
${this.crawlTemplate?.inactive || !this.crawlTemplate ? "" : html` (this.openDialogName = "config")} > ${msg("Edit")} `}

${msg("Schedule")}

${this.renderSchedule()}
${this.crawlTemplate?.inactive || !this.crawlTemplate ? "" : html` (this.openDialogName = "schedule")} > ${msg("Edit")} `}

${msg("Crawls")}

${this.renderCrawls()}
${this.renderDialogs()} `; } private renderMenu() { if (!this.crawlTemplate) return; const closeDropdown = (e: any) => { e.target.closest("sl-dropdown").hide(); }; const menuItems: HTMLTemplateResult[] = [ html` `, ]; if (!this.crawlTemplate.inactive) { menuItems.unshift(html`

`); } if (this.crawlTemplate.crawlCount && !this.crawlTemplate.inactive) { menuItems.push(html` `); } if (!this.crawlTemplate.crawlCount) { menuItems.push(html` `); } return html` ${msg("Actions")} `; } private renderInactiveNotice() { if (this.crawlTemplate?.inactive) { if (this.crawlTemplate?.newId) { return html` ${msg("This crawl template is inactive.")} ${msg("Go to newer version")} `; } return html` ${msg("This crawl template is inactive.")} `; } return ""; } private renderCurrentlyRunningNotice() { if (this.crawlTemplate?.currCrawlId) { return html` ${msg("View currently running crawl")} `; } return ""; } private renderDetails() { return html`
${msg("Created at")}
${this.crawlTemplate?.created ? html` ` : html``}
${msg("Created by")}
${this.crawlTemplate?.userName || this.crawlTemplate?.userid || html``}
`; } private renderEditName() { if (!this.crawlTemplate) return; return html`
(this.openDialogName = undefined)} >${msg("Cancel")} ${msg("Save Changes")}
`; } private renderConfiguration() { const seeds = this.crawlTemplate?.config.seeds || []; const configCodeYaml = jsonToYaml(this.crawlTemplate?.config || {}); return html`
${seeds.length > SEED_URLS_MAX ? html` (this.showAllSeedURLs = !this.showAllSeedURLs)} > ${this.showAllSeedURLs ? msg("Show less") : msg(str`Show ${seeds.length - SEED_URLS_MAX} more`)} ` : ""}
${msg("Include External Links")}
${this.crawlTemplate?.config.extraHops ? msg("Yes") : msg("No")}
${msg("Browser Profile")}
${this.crawlTemplate ? html` ${this.crawlTemplate.profileid ? html` ${this.crawlTemplate.profileName} ` : html`${msg("None")}`} ` : ""}
${msg("Advanced Configuration")}
${configCodeYaml}
`; } private renderEditConfiguration() { if (!this.crawlTemplate) return; const shouldReplacingTemplate = this.crawlTemplate.crawlCount > 0; return html`
${shouldReplacingTemplate ? html`

${msg( "Editing the crawl configuration will replace this crawl template with a new version. All other settings will be kept the same." )}

` : ""}

${this.isConfigCodeView ? msg("Custom Config") : msg("Crawl Configuration")}

(this.isConfigCodeView = e.target.checked)} > ${msg("Advanced Editor")}
${this.renderSeedsCodeEditor()}
${this.renderSeedsForm()}
(this.openDialogName = undefined)} >${msg("Cancel")} ${msg("Save Changes")}
`; } private renderSchedule() { return html`
${msg("Recurring crawls")}
${this.crawlTemplate ? html` ${this.crawlTemplate.schedule ? // TODO localize // NOTE human-readable string is in UTC, limitation of library // currently being used. // https://github.com/bradymholt/cRonstrue/issues/94 html`${cronstrue.toString(this.crawlTemplate.schedule, { verbose: true, })} (in UTC time zone)` : html`${msg("None")}`} ` : html``}
`; } private renderEditSchedule() { if (!this.crawlTemplate) return; return html` (this.openDialogName = undefined)} @submit=${this.handleSubmitEditSchedule} > `; } private renderCrawls() { return html`
${msg("# of Crawls")}
${(this.crawlTemplate?.crawlCount || 0).toLocaleString()}
${msg("Crawl Scale")}
${this.crawlTemplate?.scale} ${!this.crawlTemplate || this.crawlTemplate.inactive ? "" : html` `}
${msg("Currently Running Crawl")}
${this.crawlTemplate ? html` ${this.crawlTemplate.currCrawlId ? html` ${msg("View crawl")}` : this.crawlTemplate.inactive ? "" : html`${msg("None")}`} ` : html` `}
${msg("Latest Crawl")}
${this.crawlTemplate?.lastCrawlId ? html` ${this.crawlTemplate.lastCrawlState.replace(/_/g, " ")} ${msg("View crawl")} ` : html`${msg("None")}`}
`; } private renderEditScale() { if (!this.crawlTemplate) return; return html` ${msg("Standard")} ${msg("Big (2x)")} ${msg("Bigger (3x)")}
(this.openDialogName = undefined)} >${msg("Cancel")} ${msg("Save Changes")}
`; } /** * Render dialog for edit forms * * `openDialogName` shows/hides a specific dialog, while `isDialogVisible` * renders/prevents rendering a dialog's content unless the dialog is visible * in order to reset the dialog content on close. */ private renderDialogs() { const dialogWidth = "36rem"; const resetScroll = (e: any) => { const dialogBody = e.target.shadowRoot.querySelector('[part="body"]'); if (dialogBody) { dialogBody.scrollTop = 0; } }; return html` (this.openDialogName = undefined)} @sl-show=${() => (this.isDialogVisible = true)} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${this.isDialogVisible ? this.renderEditName() : ""} (this.openDialogName = undefined)} @sl-show=${() => (this.isDialogVisible = true)} @sl-after-hide=${() => (this.isDialogVisible = false)} @sl-after-show=${resetScroll} > ${this.isDialogVisible ? this.renderEditConfiguration() : ""} (this.openDialogName = undefined)} @sl-show=${() => (this.isDialogVisible = true)} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${this.isDialogVisible ? this.renderEditSchedule() : ""} (this.openDialogName = undefined)} @sl-show=${() => (this.isDialogVisible = true)} @sl-after-hide=${() => (this.isDialogVisible = false)} > ${this.isDialogVisible ? this.renderEditScale() : ""} `; } private renderSeedsForm() { return html` Page Page SPA Prefix Host Domain Any ${msg("Include External Links (“one hop out”)")} `; } private renderSeedsCodeEditor() { return html` { this.configCode = e.detail.value; }} > `; } async getCrawlTemplate(): Promise { const data: CrawlTemplate = await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/${this.crawlConfigId}`, this.authState! ); return data; } /** * Create a new template using existing template data */ private async duplicateConfig() { if (!this.crawlTemplate) return; const config: CrawlTemplate["config"] = { ...this.crawlTemplate.config, }; this.navTo(`/archives/${this.archiveId}/crawl-templates/new`, { crawlTemplate: { name: msg(str`${this.crawlTemplate.name} Copy`), config, }, }); this.notify({ message: msg(str`Copied crawl configuration to new template.`), type: "success", icon: "check2-circle", }); } private async handleSubmitEditName(e: { detail: { formData: FormData } }) { const { formData } = e.detail; const name = formData.get("name") as string; await this.updateTemplate({ name }); this.openDialogName = undefined; } private async handleSubmitEditScale(e: { detail: { formData: FormData } }) { const { formData } = e.detail; const scale = formData.get("scale") as string; await this.updateTemplate({ scale: +scale }); this.openDialogName = undefined; } private async handleSubmitEditConfiguration(e: { detail: { formData: FormData }; }) { const { formData } = e.detail; let config: CrawlConfig; if (this.isConfigCodeView) { if (!this.configCode) return; config = yamlToJson(this.configCode) as CrawlConfig; } else { const pageLimit = formData.get("limit") as string; const seedUrlsStr = formData.get("seedUrls") as string; config = { seeds: seedUrlsStr.trim().replace(/,/g, " ").split(/\s+/g), scopeType: formData.get("scopeType") as string, limit: pageLimit ? +pageLimit : 0, extraHops: formData.get("extraHopsOne") ? 1 : 0, }; } if (config) { await this.createRevisedTemplate(config); } this.openDialogName = undefined; } private async handleSubmitEditSchedule(e: { detail: { formData: FormData }; }) { const { formData } = e.detail; const interval = formData.get("scheduleInterval"); let schedule = ""; if (interval) { schedule = getUTCSchedule({ interval: formData.get("scheduleInterval") as any, hour: formData.get("scheduleHour") as any, minute: formData.get("scheduleMinute") as any, period: formData.get("schedulePeriod") as any, }); } await this.updateTemplate({ schedule }); this.openDialogName = undefined; } private async deactivateTemplate(): Promise { if (!this.crawlTemplate) return; try { await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/${this.crawlTemplate.id}`, this.authState!, { method: "DELETE", } ); this.notify({ message: msg( html`Deactivated ${this.crawlTemplate.name}.` ), type: "success", icon: "check2-circle", }); } catch { this.notify({ message: msg("Sorry, couldn't deactivate crawl template at this time."), type: "danger", icon: "exclamation-octagon", }); } } private async deleteTemplate(): Promise { if (!this.crawlTemplate) return; const isDeactivating = this.crawlTemplate.crawlCount > 0; try { await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/${this.crawlTemplate.id}`, this.authState!, { method: "DELETE", } ); this.navTo(`/archives/${this.archiveId}/crawl-templates`); this.notify({ message: isDeactivating ? msg(html`Deactivated ${this.crawlTemplate.name}.`) : msg(html`Deleted ${this.crawlTemplate.name}.`), type: "success", icon: "check2-circle", }); } catch { this.notify({ message: isDeactivating ? msg("Sorry, couldn't deactivate crawl template at this time.") : msg("Sorry, couldn't delete crawl template at this time."), type: "danger", icon: "exclamation-octagon", }); } } private async runNow(): Promise { try { const data = await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/${ this.crawlTemplate!.id }/run`, this.authState!, { method: "POST", } ); const crawlId = data.started; this.crawlTemplate = { ...this.crawlTemplate, currCrawlId: crawlId, } as CrawlTemplate; this.notify({ message: msg( html`Started crawl from ${this.crawlTemplate!.name}.
View crawl` ), type: "success", icon: "check2-circle", duration: 8000, }); } catch { this.notify({ message: msg("Sorry, couldn't run crawl at this time."), type: "danger", icon: "exclamation-octagon", }); } } /** * Create new crawl template with revised crawl configuration * @param config Crawl config object */ private async createRevisedTemplate(config: CrawlConfig) { this.isSubmittingUpdate = true; const params = { oldId: this.crawlTemplate!.id, name: this.crawlTemplate!.name, schedule: this.crawlTemplate!.schedule, // runNow: this.crawlTemplate!.runNow, // crawlTimeout: this.crawlTemplate!.crawlTimeout, config, }; try { const data = await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/`, this.authState!, { method: "POST", body: JSON.stringify(params), } ); console.log(data); this.navTo( `/archives/${this.archiveId}/crawl-templates/config/${data.added}` ); this.notify({ message: msg("Crawl template updated."), type: "success", icon: "check2-circle", }); } catch (e: any) { console.error(e); this.notify({ message: msg("Something went wrong, couldn't update crawl template."), type: "danger", icon: "exclamation-octagon", }); } this.isSubmittingUpdate = false; } /** * Update crawl template properties * @param params Crawl template properties to update */ private async updateTemplate(params: Partial): Promise { console.log(params); this.isSubmittingUpdate = true; try { const data = await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/${this.crawlTemplate!.id}`, this.authState!, { method: "PATCH", body: JSON.stringify(params), } ); if (data.success === true) { this.crawlTemplate = { ...this.crawlTemplate!, ...params, }; this.notify({ message: msg("Successfully saved changes."), type: "success", icon: "check2-circle", }); } else { throw data; } } catch (e: any) { console.error(e); this.notify({ message: msg("Something went wrong, couldn't update crawl template."), type: "danger", icon: "exclamation-octagon", }); } this.isSubmittingUpdate = false; } /** * Stop propgation of sl-select events. * Prevents bug where sl-dialog closes when dropdown closes * https://github.com/shoelace-style/shoelace/issues/170 */ private stopProp(e: CustomEvent) { e.stopPropagation(); } } customElements.define("btrix-crawl-templates-detail", CrawlTemplatesDetail);