import { html as staticHtml, unsafeStatic } from "lit/static-html.js"; import { state, property } from "lit/decorators.js"; import { when } from "lit/directives/when.js"; import { msg, localized, str } from "@lit/localize"; import RegexColorize from "regex-colorize"; import ISO6391 from "iso-639-1"; import LiteElement, { html } from "../utils/LiteElement"; import type { CrawlConfig, Seed, SeedConfig } from "../pages/org/types"; import { humanizeSchedule } from "../utils/cron"; import { RelativeDuration } from "./relative-duration"; /** * Usage: * ```ts * * ``` */ @localized() export class ConfigDetails extends LiteElement { @property({ type: Object }) crawlConfig?: CrawlConfig; @property({ type: Boolean }) anchorLinks = false; // Hide tag field, e.g. if embedded in crawl detail view @property({ type: Boolean }) hideTags = false; @state() private orgDefaults?: { pageLoadTimeoutSeconds?: number; behaviorTimeoutSeconds?: number; maxPagesPerCrawl?: number; }; private readonly scopeTypeLabels: Record< CrawlConfig["config"]["scopeType"], string > = { prefix: msg("Path Begins with This URL"), host: msg("Pages on This Domain"), domain: msg("Pages on This Domain & Subdomains"), "page-spa": msg("Single Page App (In-Page Links Only)"), page: msg("Page"), custom: msg("Custom"), any: msg("Any"), }; connectedCallback() { super.connectedCallback(); this.fetchAPIDefaults(); } render() { const crawlConfig = this.crawlConfig; const seedsConfig = crawlConfig?.config; const exclusions = seedsConfig?.exclude || []; const maxPages = seedsConfig?.seeds[0]?.limit ?? seedsConfig?.limit; const renderTimeLimit = ( valueSeconds?: number | null, fallbackValue?: number ) => { if (valueSeconds) { return RelativeDuration.humanize(valueSeconds * 1000, { verbose: true, }); } if (typeof fallbackValue === "number") { let value = ""; if (fallbackValue === Infinity) { value = msg("Unlimited"); } else if (fallbackValue === 0) { value = msg("0 seconds"); } else { value = RelativeDuration.humanize(fallbackValue * 1000, { verbose: true, }); } return html`${value} ${msg("(default)")}`; } }; return html`

${msg("Crawler Settings")}

${this.renderAnchorLink("crawler-settings")}
${when( crawlConfig?.jobType === "seed-crawl", this.renderConfirmSeededSettings, this.renderConfirmUrlListSettings )} ${when( exclusions.length, () => html`
`, () => this.renderSetting(msg("Exclusions"), msg("None")) )} ${this.renderSetting( msg("Max Pages"), when( maxPages, () => msg(str`${maxPages!.toLocaleString()} pages`), () => this.orgDefaults?.maxPagesPerCrawl ? html`${msg( str`${this.orgDefaults.maxPagesPerCrawl.toLocaleString()} pages` )} ${msg("(default)")}` : undefined ) )} ${this.renderSetting( msg("Page Load Timeout"), renderTimeLimit( crawlConfig?.config.pageLoadTimeout, this.orgDefaults?.pageLoadTimeoutSeconds ?? Infinity ) )} ${this.renderSetting( msg("Page Behavior Timeout"), renderTimeLimit( crawlConfig?.config.behaviorTimeout, this.orgDefaults?.behaviorTimeoutSeconds ?? Infinity ) )} ${this.renderSetting( msg("Auto-Scroll Behavior"), crawlConfig?.config.behaviors && !crawlConfig.config.behaviors.includes("autoscroll") ? msg("Disabled") : html`${msg("Enabled (default)")}` )} ${this.renderSetting( msg("Delay Before Next Page"), renderTimeLimit(crawlConfig?.config.pageExtraDelay, 0) )} ${this.renderSetting( msg("Crawl Time Limit"), renderTimeLimit(crawlConfig?.crawlTimeout, Infinity) )} ${this.renderSetting(msg("Crawler Instances"), crawlConfig?.scale)}

${msg("Browser Settings")}

${this.renderAnchorLink("browser-settings")}
${this.renderSetting( msg("Browser Profile"), when( crawlConfig?.profileid, () => html` ${crawlConfig?.profileName} `, () => msg("Default Profile") ) )} ${this.renderSetting( msg("Block Ads by Domain"), crawlConfig?.config.blockAds )} ${this.renderSetting( msg("Language"), ISO6391.getName(crawlConfig?.config.lang!) )}

${msg("Crawl Scheduling")}

${this.renderAnchorLink("crawl-scheduling")}
${this.renderSetting( msg("Crawl Schedule Type"), crawlConfig?.schedule ? msg("Run on a Recurring Basis") : msg("No Schedule") )} ${when(crawlConfig?.schedule, () => this.renderSetting( msg("Schedule"), crawlConfig?.schedule ? humanizeSchedule(crawlConfig.schedule) : undefined ) )}

${msg("Crawl Metadata")}

${this.renderAnchorLink("crawl-metadata")}
${this.renderSetting(msg("Name"), crawlConfig?.name)} ${this.renderSetting( msg("Description"), html`

${crawlConfig?.description}

` )} ${this.hideTags ? "" : this.renderSetting( msg("Tags"), crawlConfig?.tags?.length ? crawlConfig.tags.map( (tag) => html`${tag}` ) : undefined )}
`; } private renderConfirmUrlListSettings = () => { const crawlConfig = this.crawlConfig; return html` ${this.renderSetting( msg("List of URLs"), html` `, true )} ${this.renderSetting( msg("Include Any Linked Page"), Boolean(crawlConfig?.config.extraHops) )} `; }; private renderConfirmSeededSettings = () => { const crawlConfig = this.crawlConfig!; const seedsConfig = crawlConfig.config; const additionalUrlList = seedsConfig.seeds.slice(1); const primarySeedConfig: SeedConfig | Seed = seedsConfig.seeds[0]; const primarySeedUrl = primarySeedConfig.url; const includeUrlList = primarySeedConfig.include || seedsConfig.include || []; return html` ${this.renderSetting(msg("Primary Seed URL"), primarySeedUrl, true)} ${this.renderSetting( msg("Crawl Scope"), this.scopeTypeLabels[ primarySeedConfig.scopeType || seedsConfig.scopeType ] )} ${this.renderSetting( msg("Extra URLs in Scope"), includeUrlList?.length ? html` ` : msg("None"), true )} ${when( ["host", "domain", "custom", "any"].includes( primarySeedConfig.scopeType || seedsConfig.scopeType ), () => this.renderSetting( msg("Max Depth"), primarySeedConfig.depth ? msg(str`${primarySeedConfig.depth} hop(s)`) : msg("None") ) )} ${this.renderSetting( msg("Include Any Linked Page (“one hop out”)"), Boolean(primarySeedConfig.extraHops ?? seedsConfig.extraHops) )} ${this.renderSetting( msg("List of Additional URLs"), additionalUrlList?.length ? html` ` : msg("None"), true )} `; }; private renderAnchorLink(id: string) { if (!this.anchorLinks) return; const currentUrl = window.location.href; return html` `; } private renderSetting(label: string, value: any, breakAll?: boolean) { let content = value; if (!this.crawlConfig) { content = html` `; } else if (typeof value === "boolean") { content = value ? msg("Yes") : msg("No"); } else if (typeof value !== "number" && !value) { content = html`${msg("Not specified")}`; } return html` ${content} `; } private async fetchAPIDefaults() { try { const resp = await fetch("/api/settings", { headers: { "Content-Type": "application/json" }, }); if (!resp.ok) { throw new Error(resp.statusText); } const orgDefaults = { ...this.orgDefaults, }; const data = await resp.json(); if (data.defaultBehaviorTimeSeconds > 0) { orgDefaults.behaviorTimeoutSeconds = data.defaultBehaviorTimeSeconds; } if (data.defaultPageLoadTimeSeconds > 0) { orgDefaults.pageLoadTimeoutSeconds = data.defaultPageLoadTimeSeconds; } if (data.maxPagesPerCrawl > 0) { orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl; } this.orgDefaults = orgDefaults; } catch (e: any) { console.debug(e); } } }