Edit crawl config as YAML (#207)
This commit is contained in:
sua yoo
GitHub
parent
9a6483630e
commit
29b586b03f
@ -19,7 +19,8 @@
|
||||
"lodash": "^4.17.21",
|
||||
"path-parser": "^6.1.0",
|
||||
"pretty-ms": "^7.0.1",
|
||||
"tailwindcss": "^3.0.15"
|
||||
"tailwindcss": "^3.0.15",
|
||||
"yaml": "^2.0.0-11"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "web-test-runner \"src/**/*.test.{ts,js}\" --node-resolve --playwright --browsers chromium",
|
||||
|
||||
185
frontend/src/components/config-editor.ts
Normal file
185
frontend/src/components/config-editor.ts
Normal file
@ -0,0 +1,185 @@
|
||||
import { property, state, query } from "lit/decorators.js";
|
||||
import { msg, localized } from "@lit/localize";
|
||||
import {
|
||||
parse as yamlToJson,
|
||||
stringify as yamlStringify,
|
||||
YAMLParseError,
|
||||
} from "yaml";
|
||||
|
||||
import LiteElement, { html } from "../utils/LiteElement";
|
||||
|
||||
/**
|
||||
* Usage example:
|
||||
* ```ts
|
||||
* <btrix-config-editor
|
||||
* value=${value}
|
||||
* @on-change=${handleChange}
|
||||
* >
|
||||
* </btrix-config-editor>
|
||||
* ```
|
||||
*
|
||||
* @event on-change
|
||||
*/
|
||||
@localized()
|
||||
export class ConfigEditor extends LiteElement {
|
||||
@property({ type: String })
|
||||
value = "";
|
||||
|
||||
@state()
|
||||
errorMessage = "";
|
||||
|
||||
@query("#config-editor-textarea")
|
||||
textareaElem?: HTMLTextAreaElement;
|
||||
|
||||
render() {
|
||||
return html`
|
||||
<article class="border rounded">
|
||||
<header
|
||||
class="flex items-center justify-between bg-neutral-50 border-b p-1"
|
||||
>
|
||||
<div class="px-1">
|
||||
${this.errorMessage
|
||||
? html`
|
||||
<sl-icon
|
||||
class="text-danger inline-block align-middle mr-1"
|
||||
name="x-octagon"
|
||||
></sl-icon>
|
||||
<span
|
||||
class="inline-block align-middle text-sm text-neutral-500"
|
||||
>${msg("Invalid Configuration")}</span
|
||||
>
|
||||
`
|
||||
: html`
|
||||
<sl-icon
|
||||
class="text-success inline-block align-middle mr-1"
|
||||
name="check2"
|
||||
></sl-icon>
|
||||
<span
|
||||
class="inline-block align-middle text-sm text-neutral-500"
|
||||
>${msg("Valid Configuration")}</span
|
||||
>
|
||||
`}
|
||||
</div>
|
||||
|
||||
<btrix-copy-button
|
||||
.getValue=${() => this.textareaElem?.value}
|
||||
></btrix-copy-button>
|
||||
</header>
|
||||
|
||||
${this.renderTextArea()}
|
||||
|
||||
<div class="text-sm">
|
||||
${this.errorMessage
|
||||
? html`<btrix-alert type="danger">
|
||||
<div class="whitespace-pre-wrap">${this.errorMessage}</div>
|
||||
</btrix-alert> `
|
||||
: ""}
|
||||
</div>
|
||||
</article>
|
||||
`;
|
||||
}
|
||||
|
||||
private renderTextArea() {
|
||||
const lineCount = this.value.split("\n").length;
|
||||
|
||||
return html`
|
||||
<div class="flex font-mono text-sm leading-relaxed py-2">
|
||||
<div class="shrink-0 w-12 px-2 text-right text-neutral-300">
|
||||
${[...new Array(lineCount)].map((line, i) => html`${i + 1}<br />`)}
|
||||
</div>
|
||||
<div class="flex-1 px-2 overflow-x-auto text-slate-600">
|
||||
<textarea
|
||||
name="config"
|
||||
id="config-editor-textarea"
|
||||
class="language-yaml block w-full h-full overflow-y-hidden outline-none resize-none"
|
||||
autocomplete="off"
|
||||
autocapitalize="off"
|
||||
spellcheck="false"
|
||||
wrap="off"
|
||||
rows=${lineCount}
|
||||
.value=${this.value}
|
||||
@keydown=${(e: any) => {
|
||||
const textarea = e.target;
|
||||
|
||||
// Add indentation when pressing tab key instead of moving focus
|
||||
if (e.keyCode === /* tab: */ 9) {
|
||||
e.preventDefault();
|
||||
|
||||
textarea.setRangeText(
|
||||
" ",
|
||||
textarea.selectionStart,
|
||||
textarea.selectionStart,
|
||||
"end"
|
||||
);
|
||||
}
|
||||
}}
|
||||
@change=${(e: any) => {
|
||||
e.stopPropagation();
|
||||
this.onChange((e.target as HTMLTextAreaElement).value);
|
||||
}}
|
||||
@blur=${(e: any) => {
|
||||
e.stopPropagation();
|
||||
this.onBlur((e.target as HTMLTextAreaElement).value);
|
||||
}}
|
||||
@paste=${(e: any) => {
|
||||
// Use timeout to get value after paste
|
||||
window.setTimeout(() => {
|
||||
this.onChange((e.target as HTMLTextAreaElement).value);
|
||||
});
|
||||
}}
|
||||
></textarea>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
private handleParseError(error: Error) {
|
||||
if (error instanceof YAMLParseError) {
|
||||
const errorMessage = error.message.replace("YAMLParseError: ", "");
|
||||
this.errorMessage = errorMessage;
|
||||
} else {
|
||||
this.errorMessage = msg("Invalid YAML or JSON");
|
||||
console.debug(error);
|
||||
}
|
||||
}
|
||||
|
||||
private checkValidity(value: string) {
|
||||
yamlToJson(value);
|
||||
}
|
||||
|
||||
private onBlur(value: string) {
|
||||
if (!value) {
|
||||
this.textareaElem?.setCustomValidity(msg("Please fill out this field"));
|
||||
this.textareaElem?.reportValidity();
|
||||
}
|
||||
}
|
||||
|
||||
private onChange(value: string) {
|
||||
try {
|
||||
this.checkValidity(value);
|
||||
this.textareaElem?.setCustomValidity("");
|
||||
this.errorMessage = "";
|
||||
this.dispatchEvent(
|
||||
new CustomEvent("on-change", {
|
||||
detail: {
|
||||
value: value,
|
||||
},
|
||||
})
|
||||
);
|
||||
} catch (e: any) {
|
||||
this.textareaElem?.setCustomValidity(msg("Please fix errors"));
|
||||
this.handleParseError(e);
|
||||
}
|
||||
|
||||
this.textareaElem?.reportValidity();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop propgation of sl-select events.
|
||||
* Prevents bug where sl-dialog closes when dropdown closes
|
||||
* https://github.com/shoelace-style/shoelace/issues/170
|
||||
*/
|
||||
private stopProp(e: CustomEvent) {
|
||||
e.stopPropagation();
|
||||
}
|
||||
}
|
||||
@ -7,7 +7,11 @@ import { msg, localized } from "@lit/localize";
|
||||
*
|
||||
* Usage example:
|
||||
* ```ts
|
||||
* <btrix-copy-button .value=${value} @on-copied=${console.log}></btrix-copy-button>
|
||||
* <btrix-copy-button .value=${value}></btrix-copy-button>
|
||||
* ```
|
||||
* Or:
|
||||
* ```ts
|
||||
* <btrix-copy-button .getValue=${() => value}></btrix-copy-button>
|
||||
* ```
|
||||
*
|
||||
* @event on-copied
|
||||
@ -17,6 +21,9 @@ export class CopyButton extends LitElement {
|
||||
@property({ type: String })
|
||||
value?: string;
|
||||
|
||||
@property({ type: Function })
|
||||
getValue?: () => string;
|
||||
|
||||
@state()
|
||||
private isCopied: boolean = false;
|
||||
|
||||
@ -33,18 +40,22 @@ export class CopyButton extends LitElement {
|
||||
|
||||
render() {
|
||||
return html`
|
||||
<sl-button size="small" @click=${this.onClick} ?disabled=${!this.value}
|
||||
<sl-button
|
||||
size="small"
|
||||
@click=${this.onClick}
|
||||
?disabled=${!this.value && !this.getValue}
|
||||
>${this.isCopied ? msg("Copied") : msg("Copy")}</sl-button
|
||||
>
|
||||
`;
|
||||
}
|
||||
|
||||
private onClick() {
|
||||
CopyButton.copyToClipboard(this.value!);
|
||||
const value = (this.getValue ? this.getValue() : this.value) || "";
|
||||
CopyButton.copyToClipboard(value);
|
||||
|
||||
this.isCopied = true;
|
||||
|
||||
this.dispatchEvent(new CustomEvent("on-copied", { detail: this.value }));
|
||||
this.dispatchEvent(new CustomEvent("on-copied", { detail: value }));
|
||||
|
||||
this.timeoutId = window.setTimeout(() => {
|
||||
this.isCopied = false;
|
||||
|
||||
@ -9,6 +9,9 @@ import("./account-settings").then(({ AccountSettings }) => {
|
||||
import("./archive-invite-form").then(({ ArchiveInviteForm }) => {
|
||||
customElements.define("btrix-archive-invite-form", ArchiveInviteForm);
|
||||
});
|
||||
import("./config-editor").then(({ ConfigEditor }) => {
|
||||
customElements.define("btrix-config-editor", ConfigEditor);
|
||||
});
|
||||
import("./archives-list").then(({ ArchivesList }) => {
|
||||
customElements.define("btrix-archives-list", ArchivesList);
|
||||
});
|
||||
|
||||
@ -3,6 +3,7 @@ import { state, property } from "lit/decorators.js";
|
||||
import { ifDefined } from "lit/directives/if-defined.js";
|
||||
import { msg, localized, str } from "@lit/localize";
|
||||
import cronstrue from "cronstrue"; // TODO localize
|
||||
import { parse as yamlToJson, stringify as jsonToYaml } from "yaml";
|
||||
|
||||
import type { AuthState } from "../../utils/AuthService";
|
||||
import LiteElement, { html } from "../../utils/LiteElement";
|
||||
@ -36,13 +37,11 @@ export class CrawlTemplatesDetail extends LiteElement {
|
||||
private showAllSeedURLs: boolean = false;
|
||||
|
||||
@state()
|
||||
private isSeedsJsonView: boolean = false;
|
||||
private isConfigCodeView: boolean = false;
|
||||
|
||||
/** YAML or stringified JSON config */
|
||||
@state()
|
||||
private seedsJson: string = "";
|
||||
|
||||
@state()
|
||||
private invalidSeedsJsonMessage: string = "";
|
||||
private configCode: string = "";
|
||||
|
||||
@state()
|
||||
private isSubmittingUpdate: boolean = false;
|
||||
@ -70,9 +69,9 @@ export class CrawlTemplatesDetail extends LiteElement {
|
||||
(seed: any) => typeof seed !== "string"
|
||||
);
|
||||
if (isComplexConfig) {
|
||||
this.isSeedsJsonView = true;
|
||||
this.isConfigCodeView = true;
|
||||
}
|
||||
this.seedsJson = JSON.stringify(this.crawlTemplate.config, null, 2);
|
||||
this.configCode = jsonToYaml(this.crawlTemplate.config);
|
||||
} catch (e: any) {
|
||||
this.notify({
|
||||
message:
|
||||
@ -385,7 +384,7 @@ export class CrawlTemplatesDetail extends LiteElement {
|
||||
>${msg("Actions")}</sl-button
|
||||
>
|
||||
|
||||
<ul class="text-sm text-0-800 whitespace-nowrap" role="menu">
|
||||
<ul class="text-left text-sm text-0-800 whitespace-nowrap" role="menu">
|
||||
${menuItems.map((item: HTMLTemplateResult) => item)}
|
||||
</ul>
|
||||
</sl-dropdown>
|
||||
@ -590,16 +589,12 @@ export class CrawlTemplatesDetail extends LiteElement {
|
||||
|
||||
<sl-details style="--sl-spacing-medium: var(--sl-spacing-small)">
|
||||
<span slot="summary" class="text-sm">
|
||||
<span class="font-medium">${msg("JSON Configuration")}</span>
|
||||
<span class="font-medium">${msg("Advanced Configuration")}</span>
|
||||
</span>
|
||||
<div class="relative">
|
||||
<pre
|
||||
class="language-json bg-gray-800 text-gray-50 p-4 rounded font-mono text-xs overflow-auto"
|
||||
><code>${JSON.stringify(
|
||||
this.crawlTemplate?.config || {},
|
||||
null,
|
||||
2
|
||||
)}</code></pre>
|
||||
class="language-yaml text-neutral-600 p-4 rounded font-mono leading-relaxed text-xs overflow-auto"
|
||||
><code>${jsonToYaml(this.crawlTemplate?.config || {})}</code></pre>
|
||||
|
||||
<div class="absolute top-2 right-2">
|
||||
<btrix-copy-button
|
||||
@ -637,23 +632,23 @@ export class CrawlTemplatesDetail extends LiteElement {
|
||||
|
||||
<div class="flex flex-wrap justify-between">
|
||||
<h4 class="font-medium">
|
||||
${this.isSeedsJsonView
|
||||
${this.isConfigCodeView
|
||||
? msg("Custom Config")
|
||||
: msg("Crawl Configuration")}
|
||||
</h4>
|
||||
<sl-switch
|
||||
?checked=${this.isSeedsJsonView}
|
||||
?checked=${this.isConfigCodeView}
|
||||
@sl-change=${(e: any) =>
|
||||
(this.isSeedsJsonView = e.target.checked)}
|
||||
(this.isConfigCodeView = e.target.checked)}
|
||||
>
|
||||
<span class="text-sm">${msg("Use JSON Editor")}</span>
|
||||
<span class="text-sm">${msg("Advanced Editor")}</span>
|
||||
</sl-switch>
|
||||
</div>
|
||||
|
||||
<div class="${this.isSeedsJsonView ? "" : "hidden"}">
|
||||
${this.renderSeedsJson()}
|
||||
<div class="${this.isConfigCodeView ? "" : "hidden"}">
|
||||
${this.renderSeedsCodeEditor()}
|
||||
</div>
|
||||
<div class="grid gap-5${this.isSeedsJsonView ? " hidden" : ""}">
|
||||
<div class="grid gap-5${this.isConfigCodeView ? " hidden" : ""}">
|
||||
${this.renderSeedsForm()}
|
||||
</div>
|
||||
|
||||
@ -666,8 +661,7 @@ export class CrawlTemplatesDetail extends LiteElement {
|
||||
<sl-button
|
||||
type="primary"
|
||||
submit
|
||||
?disabled=${Boolean(this.invalidSeedsJsonMessage) ||
|
||||
this.isSubmittingUpdate}
|
||||
?disabled=${this.isSubmittingUpdate}
|
||||
?loading=${this.isSubmittingUpdate}
|
||||
>${msg("Save Changes")}</sl-button
|
||||
>
|
||||
@ -952,7 +946,7 @@ export class CrawlTemplatesDetail extends LiteElement {
|
||||
)}
|
||||
rows="3"
|
||||
value=${this.crawlTemplate!.config.seeds.join("\n")}
|
||||
required
|
||||
?required=${!this.isConfigCodeView}
|
||||
></sl-textarea>
|
||||
<sl-select
|
||||
name="scopeType"
|
||||
@ -986,96 +980,17 @@ export class CrawlTemplatesDetail extends LiteElement {
|
||||
`;
|
||||
}
|
||||
|
||||
private renderSeedsJson() {
|
||||
private renderSeedsCodeEditor() {
|
||||
return html`
|
||||
<div class="grid gap-4">
|
||||
<div>
|
||||
<p class="mb-2">
|
||||
${msg(
|
||||
html`See
|
||||
<a
|
||||
href="https://github.com/webrecorder/browsertrix-crawler#crawling-configuration-options"
|
||||
class="text-primary hover:underline"
|
||||
target="_blank"
|
||||
>Browsertrix Crawler docs
|
||||
<sl-icon name="box-arrow-up-right"></sl-icon
|
||||
></a>
|
||||
for all configuration options.`
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-3 gap-4">
|
||||
<div class="relative col-span-3 md:col-span-2">
|
||||
${this.renderSeedsJsonInput()}
|
||||
|
||||
<div class="absolute top-2 right-2">
|
||||
<btrix-copy-button .value=${this.seedsJson}></btrix-copy-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-span-3 md:col-span-1">
|
||||
${this.invalidSeedsJsonMessage
|
||||
? html`<btrix-alert type="danger">
|
||||
${this.invalidSeedsJsonMessage}
|
||||
</btrix-alert> `
|
||||
: html` <btrix-alert> ${msg("Valid JSON")} </btrix-alert>`}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
private renderSeedsJsonInput() {
|
||||
return html`
|
||||
<textarea
|
||||
id="json-editor"
|
||||
name="config"
|
||||
class="language-json block w-full bg-gray-800 text-gray-50 p-4 rounded font-mono text-sm"
|
||||
autocomplete="off"
|
||||
rows="10"
|
||||
spellcheck="false"
|
||||
.value=${this.seedsJson}
|
||||
@keydown=${(e: any) => {
|
||||
// Add indentation when pressing tab key instead of moving focus
|
||||
if (e.keyCode === /* tab: */ 9) {
|
||||
e.preventDefault();
|
||||
|
||||
const textarea = e.target;
|
||||
|
||||
textarea.setRangeText(
|
||||
" ",
|
||||
textarea.selectionStart,
|
||||
textarea.selectionStart,
|
||||
"end"
|
||||
);
|
||||
}
|
||||
<btrix-config-editor
|
||||
value=${this.configCode}
|
||||
@on-change=${(e: any) => {
|
||||
this.configCode = e.detail.value;
|
||||
}}
|
||||
@change=${(e: any) => (this.seedsJson = e.target.value)}
|
||||
@blur=${this.updateSeedsJson}
|
||||
></textarea>
|
||||
></btrix-config-editor>
|
||||
`;
|
||||
}
|
||||
|
||||
private updateSeedsJson(e: any) {
|
||||
const textarea = e.target;
|
||||
const text = textarea.value;
|
||||
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
|
||||
this.seedsJson = JSON.stringify(json, null, 2);
|
||||
this.invalidSeedsJsonMessage = "";
|
||||
|
||||
textarea.setCustomValidity("");
|
||||
textarea.reportValidity();
|
||||
} catch (e: any) {
|
||||
this.invalidSeedsJsonMessage = e.message
|
||||
? msg(str`JSON is invalid: ${e.message.replace("JSON.parse: ", "")}`)
|
||||
: msg("JSON is invalid.");
|
||||
}
|
||||
}
|
||||
|
||||
async getCrawlTemplate(): Promise<CrawlTemplate> {
|
||||
const data: CrawlTemplate = await this.apiFetch(
|
||||
`/archives/${this.archiveId}/crawlconfigs/${this.crawlConfigId}`,
|
||||
@ -1131,13 +1046,13 @@ export class CrawlTemplatesDetail extends LiteElement {
|
||||
detail: { formData: FormData };
|
||||
}) {
|
||||
const { formData } = e.detail;
|
||||
const configValue = formData.get("config") as string;
|
||||
|
||||
let config: CrawlConfig;
|
||||
|
||||
if (this.isSeedsJsonView) {
|
||||
if (!configValue || this.invalidSeedsJsonMessage) return;
|
||||
if (this.isConfigCodeView) {
|
||||
if (!this.configCode) return;
|
||||
|
||||
config = JSON.parse(configValue) as CrawlConfig;
|
||||
config = yamlToJson(this.configCode) as CrawlConfig;
|
||||
} else {
|
||||
const pageLimit = formData.get("limit") as string;
|
||||
const seedUrlsStr = formData.get("seedUrls") as string;
|
||||
|
||||
@ -2,6 +2,7 @@ import { state, property } from "lit/decorators.js";
|
||||
import { ifDefined } from "lit/directives/if-defined.js";
|
||||
import { msg, localized, str } from "@lit/localize";
|
||||
import cronParser from "cron-parser";
|
||||
import { parse as yamlToJson, stringify as jsonToYaml } from "yaml";
|
||||
|
||||
import type { AuthState } from "../../utils/AuthService";
|
||||
import LiteElement, { html } from "../../utils/LiteElement";
|
||||
@ -73,13 +74,11 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
};
|
||||
|
||||
@state()
|
||||
private isSeedsJsonView: boolean = false;
|
||||
private isConfigCodeView: boolean = false;
|
||||
|
||||
/** YAML or stringified JSON config */
|
||||
@state()
|
||||
private seedsJson: string = "";
|
||||
|
||||
@state()
|
||||
private invalidSeedsJsonMessage: string = "";
|
||||
private configCode: string = "";
|
||||
|
||||
@state()
|
||||
private isSubmitting: boolean = false;
|
||||
@ -126,7 +125,7 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
(seed: any) => typeof seed !== "string"
|
||||
);
|
||||
if (isComplexConfig) {
|
||||
this.isSeedsJsonView = true;
|
||||
this.isConfigCodeView = true;
|
||||
}
|
||||
this.initialCrawlTemplate = {
|
||||
name: this.initialCrawlTemplate?.name || initialValues.name,
|
||||
@ -135,7 +134,7 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
...this.initialCrawlTemplate?.config,
|
||||
},
|
||||
};
|
||||
this.seedsJson = JSON.stringify(this.initialCrawlTemplate.config, null, 2);
|
||||
this.configCode = jsonToYaml(this.initialCrawlTemplate.config);
|
||||
super.connectedCallback();
|
||||
}
|
||||
|
||||
@ -167,7 +166,7 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
<main class="mt-6">
|
||||
<div class="md:border md:rounded-lg">
|
||||
<sl-form @sl-submit=${this.onSubmit} aria-describedby="formError">
|
||||
<div class="md:grid grid-cols-3">
|
||||
<div class="grid grid-cols-3">
|
||||
${this.renderBasicSettings()} ${this.renderCrawlConfigSettings()}
|
||||
${this.renderScheduleSettings()}
|
||||
</div>
|
||||
@ -219,10 +218,10 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
|
||||
private renderBasicSettings() {
|
||||
return html`
|
||||
<div class="col-span-1 py-2 md:p-8 md:border-b">
|
||||
<div class="col-span-3 md:col-span-1 py-2 md:p-8 md:border-b">
|
||||
<h3 class="font-medium">${msg("Basic Settings")}</h3>
|
||||
</div>
|
||||
<section class="col-span-2 pb-6 md:p-8 border-b grid gap-5">
|
||||
<section class="col-span-3 md:col-span-2 pb-6 md:p-8 border-b grid gap-5">
|
||||
<sl-input
|
||||
name="name"
|
||||
label=${msg("Name")}
|
||||
@ -242,10 +241,10 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
|
||||
private renderScheduleSettings() {
|
||||
return html`
|
||||
<div class="col-span-1 py-2 md:p-8 md:border-b">
|
||||
<div class="col-span-3 md:col-span-1 py-2 md:p-8 md:border-b">
|
||||
<h3 class="font-medium">${msg("Crawl Schedule")}</h3>
|
||||
</div>
|
||||
<section class="col-span-2 pb-6 md:p-8 border-b grid gap-5">
|
||||
<section class="col-span-3 md:col-span-2 pb-6 md:p-8 border-b grid gap-5">
|
||||
<div>
|
||||
<div class="flex items-end">
|
||||
<div class="pr-2 flex-1">
|
||||
@ -358,11 +357,13 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
|
||||
private renderCrawlConfigSettings() {
|
||||
return html`
|
||||
<div class="col-span-1 py-2 md:p-8 md:border-b">
|
||||
<div class="col-span-3 md:col-span-1 py-2 md:p-8 md:border-b">
|
||||
<h3 class="font-medium">${msg("Crawl Settings")}</h3>
|
||||
</div>
|
||||
<section class="col-span-2 pb-6 md:p-8 border-b grid gap-5">
|
||||
<div>
|
||||
<section
|
||||
class="col-span-3 md:col-span-2 pb-6 md:p-8 border-b grid grid-cols-1 gap-5"
|
||||
>
|
||||
<div class="col-span-1">
|
||||
<sl-select
|
||||
name="scale"
|
||||
label=${msg("Crawl Scale")}
|
||||
@ -373,24 +374,26 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
<sl-menu-item value="3">${msg("Bigger (3x)")}</sl-menu-item>
|
||||
</sl-select>
|
||||
</div>
|
||||
<div class="flex justify-between">
|
||||
<div class="col-span-1 flex justify-between">
|
||||
<h4 class="font-medium">
|
||||
${this.isSeedsJsonView
|
||||
${this.isConfigCodeView
|
||||
? msg("Custom Config")
|
||||
: msg("Crawl Configuration")}
|
||||
</h4>
|
||||
<sl-switch
|
||||
?checked=${this.isSeedsJsonView}
|
||||
@sl-change=${(e: any) => (this.isSeedsJsonView = e.target.checked)}
|
||||
?checked=${this.isConfigCodeView}
|
||||
@sl-change=${(e: any) => (this.isConfigCodeView = e.target.checked)}
|
||||
>
|
||||
<span class="text-sm">${msg("Use JSON Editor")}</span>
|
||||
<span class="text-sm">${msg("Advanced Editor")}</span>
|
||||
</sl-switch>
|
||||
</div>
|
||||
|
||||
<div class="${this.isSeedsJsonView ? "" : "hidden"}">
|
||||
${this.renderSeedsJson()}
|
||||
<div class="col-span-1${this.isConfigCodeView ? "" : " hidden"}">
|
||||
${this.renderSeedsCodeEditor()}
|
||||
</div>
|
||||
<div class="grid gap-5${this.isSeedsJsonView ? "hidden" : ""}">
|
||||
<div
|
||||
class="col-span-1 grid gap-5${this.isConfigCodeView ? " hidden" : ""}"
|
||||
>
|
||||
${this.renderSeedsForm()}
|
||||
</div>
|
||||
</section>
|
||||
@ -410,7 +413,7 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
)}
|
||||
rows="3"
|
||||
value=${this.initialCrawlTemplate!.config.seeds.join("\n")}
|
||||
required
|
||||
?required=${!this.isConfigCodeView}
|
||||
></sl-textarea>
|
||||
<sl-select
|
||||
name="scopeType"
|
||||
@ -442,10 +445,10 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
`;
|
||||
}
|
||||
|
||||
private renderSeedsJson() {
|
||||
private renderSeedsCodeEditor() {
|
||||
return html`
|
||||
<div class="grid gap-4">
|
||||
<div>
|
||||
<div class="grid grid-cols-1 gap-4">
|
||||
<div class="col-span-1">
|
||||
<p class="mb-2">
|
||||
${msg(
|
||||
html`See
|
||||
@ -461,76 +464,17 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-3 gap-4">
|
||||
<div class="relative col-span-2">
|
||||
${this.renderSeedsJsonInput()}
|
||||
|
||||
<div class="absolute top-2 right-2">
|
||||
<btrix-copy-button .value=${this.seedsJson}></btrix-copy-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-span-1">
|
||||
${this.invalidSeedsJsonMessage
|
||||
? html`<btrix-alert type="danger">
|
||||
${this.invalidSeedsJsonMessage}
|
||||
</btrix-alert> `
|
||||
: html` <btrix-alert> ${msg("Valid JSON")} </btrix-alert>`}
|
||||
</div>
|
||||
</div>
|
||||
<btrix-config-editor
|
||||
class="col-span-1"
|
||||
value=${this.configCode}
|
||||
@on-change=${(e: any) => {
|
||||
this.configCode = e.detail.value;
|
||||
}}
|
||||
></btrix-config-editor>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
private renderSeedsJsonInput() {
|
||||
return html`
|
||||
<textarea
|
||||
id="json-editor"
|
||||
class="language-json block w-full bg-gray-800 text-gray-50 p-4 rounded font-mono text-sm"
|
||||
autocomplete="off"
|
||||
rows="10"
|
||||
spellcheck="false"
|
||||
.value=${this.seedsJson}
|
||||
@keydown=${(e: any) => {
|
||||
// Add indentation when pressing tab key instead of moving focus
|
||||
if (e.keyCode === /* tab: */ 9) {
|
||||
e.preventDefault();
|
||||
|
||||
const textarea = e.target;
|
||||
|
||||
textarea.setRangeText(
|
||||
" ",
|
||||
textarea.selectionStart,
|
||||
textarea.selectionStart,
|
||||
"end"
|
||||
);
|
||||
}
|
||||
}}
|
||||
@change=${(e: any) => (this.seedsJson = e.target.value)}
|
||||
@blur=${this.updateSeedsJson}
|
||||
></textarea>
|
||||
`;
|
||||
}
|
||||
|
||||
private updateSeedsJson(e: any) {
|
||||
const textarea = e.target;
|
||||
const text = textarea.value;
|
||||
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
|
||||
this.seedsJson = JSON.stringify(json, null, 2);
|
||||
this.invalidSeedsJsonMessage = "";
|
||||
|
||||
textarea.setCustomValidity("");
|
||||
textarea.reportValidity();
|
||||
} catch (e: any) {
|
||||
this.invalidSeedsJsonMessage = e.message
|
||||
? msg(str`JSON is invalid: ${e.message.replace("JSON.parse: ", "")}`)
|
||||
: msg("JSON is invalid.");
|
||||
}
|
||||
}
|
||||
|
||||
private parseTemplate(formData: FormData) {
|
||||
const crawlTimeoutMinutes = formData.get("crawlTimeoutMinutes");
|
||||
const pageLimit = formData.get("limit");
|
||||
@ -544,8 +488,8 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
scale: +scale,
|
||||
};
|
||||
|
||||
if (this.isSeedsJsonView) {
|
||||
template.config = JSON.parse(this.seedsJson);
|
||||
if (this.isConfigCodeView) {
|
||||
template.config = yamlToJson(this.configCode) as CrawlConfig;
|
||||
} else {
|
||||
template.config = {
|
||||
seeds: (seedUrlsStr as string).trim().replace(/,/g, " ").split(/\s+/g),
|
||||
@ -564,20 +508,8 @@ export class CrawlTemplatesNew extends LiteElement {
|
||||
}) {
|
||||
if (!this.authState) return;
|
||||
|
||||
if (this.isSeedsJsonView && this.invalidSeedsJsonMessage) {
|
||||
// Check JSON validity
|
||||
const jsonEditor = event.target.querySelector("#json-editor");
|
||||
|
||||
jsonEditor.setCustomValidity(msg("Please correct JSON errors."));
|
||||
jsonEditor.reportValidity();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const params = this.parseTemplate(event.detail.formData);
|
||||
|
||||
console.log(params);
|
||||
|
||||
this.serverError = undefined;
|
||||
this.isSubmitting = true;
|
||||
|
||||
|
||||
@ -5604,6 +5604,11 @@ yaml@^1.10.0, yaml@^1.10.2:
|
||||
resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"
|
||||
integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==
|
||||
|
||||
yaml@^2.0.0-11:
|
||||
version "2.0.0-11"
|
||||
resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.0.0-11.tgz#269af42637a41ec1ebf2abb546a28949545f0cbb"
|
||||
integrity sha512-5kGSQrzDyjCk0BLuFfjkoUE9vYcoyrwZIZ+GnpOSM9vhkvPjItYiWJ1jpRSo0aU4QmsoNrFwDT4O7XS2UGcBQg==
|
||||
|
||||
yargs-parser@^20.2.9:
|
||||
version "20.2.9"
|
||||
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user