import { html as staticHtml, unsafeStatic } from "lit/static-html.js";
import { state, property } from "lit/decorators.js";
import { when } from "lit/directives/when.js";
import { msg, localized, str } from "@lit/localize";
import RegexColorize from "regex-colorize";
import ISO6391 from "iso-639-1";
import LiteElement, { html } from "../utils/LiteElement";
import type { CrawlConfig, Seed, SeedConfig } from "../pages/org/types";
import { humanizeSchedule } from "../utils/cron";
import { RelativeDuration } from "./relative-duration";
/**
* Usage:
* ```ts
*
* ```
*/
@localized()
export class ConfigDetails extends LiteElement {
@property({ type: Object })
crawlConfig?: CrawlConfig;
@property({ type: Boolean })
anchorLinks = false;
// Hide tag field, e.g. if embedded in crawl detail view
@property({ type: Boolean })
hideTags = false;
@state()
private orgDefaults?: {
pageLoadTimeoutSeconds?: number;
behaviorTimeoutSeconds?: number;
maxPagesPerCrawl?: number;
};
private readonly scopeTypeLabels: Record<
CrawlConfig["config"]["scopeType"],
string
> = {
prefix: msg("Path Begins with This URL"),
host: msg("Pages on This Domain"),
domain: msg("Pages on This Domain & Subdomains"),
"page-spa": msg("Single Page App (In-Page Links Only)"),
page: msg("Page"),
custom: msg("Custom"),
any: msg("Any"),
};
connectedCallback() {
super.connectedCallback();
this.fetchAPIDefaults();
}
render() {
const crawlConfig = this.crawlConfig;
const seedsConfig = crawlConfig?.config;
const exclusions = seedsConfig?.exclude || [];
const maxPages = seedsConfig?.seeds[0]?.limit ?? seedsConfig?.limit;
const renderTimeLimit = (
valueSeconds?: number | null,
fallbackValue?: number
) => {
if (valueSeconds) {
return RelativeDuration.humanize(valueSeconds * 1000, {
verbose: true,
});
}
if (typeof fallbackValue === "number") {
let value = "";
if (fallbackValue === Infinity) {
value = msg("Unlimited");
} else if (fallbackValue === 0) {
value = msg("0 seconds");
} else {
value = RelativeDuration.humanize(fallbackValue * 1000, {
verbose: true,
});
}
return html`${value} ${msg("(default)")}`;
}
};
return html`
${msg("Crawler Settings")}
${this.renderAnchorLink("crawler-settings")}
${when(
crawlConfig?.jobType === "seed-crawl",
this.renderConfirmSeededSettings,
this.renderConfirmUrlListSettings
)}
${when(
exclusions.length,
() => html`
`,
() => this.renderSetting(msg("Exclusions"), msg("None"))
)}
${this.renderSetting(
msg("Max Pages"),
when(
maxPages,
() => msg(str`${maxPages!.toLocaleString()} pages`),
() =>
this.orgDefaults?.maxPagesPerCrawl
? html`${msg(
str`${this.orgDefaults.maxPagesPerCrawl.toLocaleString()} pages`
)}
${msg("(default)")}`
: undefined
)
)}
${this.renderSetting(
msg("Page Load Timeout"),
renderTimeLimit(
crawlConfig?.config.pageLoadTimeout,
this.orgDefaults?.pageLoadTimeoutSeconds ?? Infinity
)
)}
${this.renderSetting(
msg("Page Behavior Timeout"),
renderTimeLimit(
crawlConfig?.config.behaviorTimeout,
this.orgDefaults?.behaviorTimeoutSeconds ?? Infinity
)
)}
${this.renderSetting(
msg("Auto-Scroll Behavior"),
crawlConfig?.config.behaviors &&
!crawlConfig.config.behaviors.includes("autoscroll")
? msg("Disabled")
: html`${msg("Enabled (default)")}`
)}
${this.renderSetting(
msg("Delay Before Next Page"),
renderTimeLimit(crawlConfig?.config.pageExtraDelay, 0)
)}
${this.renderSetting(
msg("Crawl Time Limit"),
renderTimeLimit(crawlConfig?.crawlTimeout, Infinity)
)}
${this.renderSetting(msg("Crawler Instances"), crawlConfig?.scale)}
${msg("Browser Settings")}
${this.renderAnchorLink("browser-settings")}
${this.renderSetting(
msg("Browser Profile"),
when(
crawlConfig?.profileid,
() => html`
${crawlConfig?.profileName}
`,
() => msg("Default Profile")
)
)}
${this.renderSetting(
msg("Block Ads by Domain"),
crawlConfig?.config.blockAds
)}
${this.renderSetting(
msg("Language"),
ISO6391.getName(crawlConfig?.config.lang!)
)}
${msg("Crawl Scheduling")}
${this.renderAnchorLink("crawl-scheduling")}
${this.renderSetting(
msg("Crawl Schedule Type"),
crawlConfig?.schedule
? msg("Run on a Recurring Basis")
: msg("No Schedule")
)}
${when(crawlConfig?.schedule, () =>
this.renderSetting(
msg("Schedule"),
crawlConfig?.schedule
? humanizeSchedule(crawlConfig.schedule)
: undefined
)
)}
`;
}
private renderConfirmUrlListSettings = () => {
const crawlConfig = this.crawlConfig;
return html`
${this.renderSetting(
msg("List of URLs"),
html`
${crawlConfig?.config.seeds.map(
(seed: Seed) => html` - ${seed.url}
`
)}
`,
true
)}
${this.renderSetting(
msg("Include Any Linked Page"),
Boolean(crawlConfig?.config.extraHops)
)}
`;
};
private renderConfirmSeededSettings = () => {
const crawlConfig = this.crawlConfig!;
const seedsConfig = crawlConfig.config;
const additionalUrlList = seedsConfig.seeds.slice(1);
const primarySeedConfig: SeedConfig | Seed = seedsConfig.seeds[0];
const primarySeedUrl = primarySeedConfig.url;
const includeUrlList =
primarySeedConfig.include || seedsConfig.include || [];
return html`
${this.renderSetting(msg("Primary Seed URL"), primarySeedUrl, true)}
${this.renderSetting(
msg("Crawl Scope"),
this.scopeTypeLabels[
primarySeedConfig.scopeType || seedsConfig.scopeType
]
)}
${this.renderSetting(
msg("Extra URLs in Scope"),
includeUrlList?.length
? html`
${includeUrlList.map(
(url: string) =>
staticHtml`- ${unsafeStatic(
new RegexColorize().colorizeText(url)
)}
`
)}
`
: msg("None"),
true
)}
${when(
["host", "domain", "custom", "any"].includes(
primarySeedConfig.scopeType || seedsConfig.scopeType
),
() =>
this.renderSetting(
msg("Max Depth"),
primarySeedConfig.depth
? msg(str`${primarySeedConfig.depth} hop(s)`)
: msg("None")
)
)}
${this.renderSetting(
msg("Include Any Linked Page (“one hop out”)"),
Boolean(primarySeedConfig.extraHops ?? seedsConfig.extraHops)
)}
${this.renderSetting(
msg("List of Additional URLs"),
additionalUrlList?.length
? html`
${additionalUrlList.map(
(seed) =>
html`- ${typeof seed === "string" ? seed : seed.url}
`
)}
`
: msg("None"),
true
)}
`;
};
private renderAnchorLink(id: string) {
if (!this.anchorLinks) return;
const currentUrl = window.location.href;
return html`
`;
}
private renderSetting(label: string, value: any, breakAll?: boolean) {
let content = value;
if (!this.crawlConfig) {
content = html` `;
} else if (typeof value === "boolean") {
content = value ? msg("Yes") : msg("No");
} else if (typeof value !== "number" && !value) {
content = html`${msg("Not specified")}`;
}
return html`
${content}
`;
}
private async fetchAPIDefaults() {
try {
const resp = await fetch("/api/settings", {
headers: { "Content-Type": "application/json" },
});
if (!resp.ok) {
throw new Error(resp.statusText);
}
const orgDefaults = {
...this.orgDefaults,
};
const data = await resp.json();
if (data.defaultBehaviorTimeSeconds > 0) {
orgDefaults.behaviorTimeoutSeconds = data.defaultBehaviorTimeSeconds;
}
if (data.defaultPageLoadTimeSeconds > 0) {
orgDefaults.pageLoadTimeoutSeconds = data.defaultPageLoadTimeSeconds;
}
if (data.maxPagesPerCrawl > 0) {
orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl;
}
this.orgDefaults = orgDefaults;
} catch (e: any) {
console.debug(e);
}
}
}