From c3edb4bba43c9e0343c176aac94cf6a46ecf2392 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 18 Jan 2022 19:58:55 -0800 Subject: [PATCH] Allow user to configure crawls with JSON (#86) --- frontend/src/components/copy-button.ts | 48 ++ frontend/src/components/index.ts | 3 + frontend/src/pages/archive/crawl-templates.ts | 584 ++++++++++++------ frontend/src/shoelace.ts | 3 + frontend/src/utils/LiteElement.ts | 7 +- 5 files changed, 439 insertions(+), 206 deletions(-) create mode 100644 frontend/src/components/copy-button.ts diff --git a/frontend/src/components/copy-button.ts b/frontend/src/components/copy-button.ts new file mode 100644 index 00000000..92813b53 --- /dev/null +++ b/frontend/src/components/copy-button.ts @@ -0,0 +1,48 @@ +import { LitElement, html } from "lit"; +import { property, state } from "lit/decorators.js"; +import { msg, localized } from "@lit/localize"; + +/** + * Copy text to clipboard on click + * + * Usage example: + * ```ts + * + * ``` + * + * @event on-copied + */ +@localized() +export class CopyButton extends LitElement { + @property({ type: String }) + value?: string; + + @state() + private isCopied: boolean = false; + + timeoutId?: number; + + disconnectedCallback() { + window.clearTimeout(this.timeoutId); + } + + render() { + return html` + ${this.isCopied ? msg("Copied") : msg("Copy")} + `; + } + + private onClick() { + navigator.clipboard.writeText(this.value!); + + this.isCopied = true; + + this.dispatchEvent(new CustomEvent("on-copied", { detail: this.value })); + + this.timeoutId = window.setTimeout(() => { + this.isCopied = false; + }, 3000); + } +} diff --git a/frontend/src/components/index.ts b/frontend/src/components/index.ts index d65e13e1..fe23527f 100644 --- a/frontend/src/components/index.ts +++ b/frontend/src/components/index.ts @@ -9,6 +9,9 @@ import("./account-settings").then(({ AccountSettings }) => { import("./archive-invite-form").then(({ ArchiveInviteForm }) => { customElements.define("btrix-archive-invite-form", ArchiveInviteForm); }); +import("./copy-button").then(({ CopyButton }) => { + customElements.define("btrix-copy-button", CopyButton); +}); import("./invite-form").then(({ InviteForm }) => { customElements.define("btrix-invite-form", InviteForm); }); diff --git a/frontend/src/pages/archive/crawl-templates.ts b/frontend/src/pages/archive/crawl-templates.ts index a9172c9a..5775f522 100644 --- a/frontend/src/pages/archive/crawl-templates.ts +++ b/frontend/src/pages/archive/crawl-templates.ts @@ -6,16 +6,30 @@ import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; import { getLocaleTimeZone } from "../../utils/localization"; -type CrawlTemplate = any; // TODO +type CrawlTemplate = { + id?: string; + name: string; + schedule: string; + runNow: boolean; + crawlTimeout?: number; + config: { + seeds: string[]; + scopeType?: string; + limit?: number; + }; +}; const initialValues = { - name: `Example crawl ${Date.now()}`, // TODO remove placeholder + name: "", runNow: true, - // crawlTimeoutMinutes: 0, - seedUrls: "", - scopeType: "prefix", - // limit: 0, + schedule: "@weekly", + config: { + seeds: [], + scopeType: "prefix", + limit: 0, + }, }; +const initialSeedsJson = JSON.stringify(initialValues.config, null, 2); const hours = Array.from({ length: 12 }).map((x, i) => ({ value: i + 1, label: `${i + 1}`, @@ -54,6 +68,21 @@ export class CrawlTemplates extends LiteElement { period: "AM", }; + @state() + private isSeedsJsonView: boolean = false; + + @state() + private seedsJson: string = initialSeedsJson; + + @state() + private invalidSeedsJsonMessage: string = ""; + + @state() + private isSubmitting: boolean = false; + + @state() + private serverError?: string; + private get timeZone() { return Intl.DateTimeFormat().resolvedOptions().timeZone; } @@ -62,17 +91,10 @@ export class CrawlTemplates extends LiteElement { return getLocaleTimeZone(); } - render() { - if (this.isNew) { - return this.renderNew(); - } - - return this.renderList(); - } - - private renderNew() { + private get nextScheduledCrawlMessage() { const utcSchedule = this.getUTCSchedule(); - const nextScheduledCrawlMessage = this.scheduleInterval + + return this.scheduleInterval ? msg(html`Next scheduled crawl: ${msg("New Crawl Template")}

@@ -101,184 +133,31 @@ export class CrawlTemplates extends LiteElement {

- -
-
-

${msg("Basic settings")}

+
+ +
+ ${this.renderBasicSettings()} ${this.renderPagesSettings()}
-
-
- -
-
-
-
- - (this.scheduleInterval = e.target.value)} - > - ${msg("None")} - ${msg("Daily")} - ${msg("Weekly")} - ${msg("Monthly")} - -
-
- ${msg("at")} - - (this.scheduleTime = { - ...this.scheduleTime, - hour: +e.target.value, - })} - > - ${hours.map( - ({ value, label }) => - html`${label}` - )} - - : - - (this.scheduleTime = { - ...this.scheduleTime, - minute: +e.target.value, - })} - > - ${minutes.map( - ({ value, label }) => - html`${label}` - )} - - - (this.scheduleTime = { - ...this.scheduleTime, - period: e.target.value, - })} - > - ${msg("AM", { desc: "Time AM/PM" })} - ${msg("PM", { desc: "Time AM/PM" })} - - ${this.timeZoneShortName} -
-
-
- ${nextScheduledCrawlMessage || msg("No crawls scheduled")} -
-
+ +
+ ${this.serverError + ? html`${this.serverError}` + : ""}
- (this.isRunNow = e.target.checked)} - >${msg("Run immediately on save")}${msg("Save Crawl Template")}
-
- - ${msg("minutes")} - -
-
- -
-

${msg("Pages")}

-
-
-
- -
-
- - Page - Page SPA - Prefix - Host - Any - -
-
- - ${msg("pages")} - -
-
- -
- ${msg("Save Crawl Template")} - ${this.isRunNow || this.scheduleInterval - ? html`
+ ? html`
${this.isRunNow ? html`

@@ -286,12 +165,12 @@ export class CrawlTemplates extends LiteElement {

` : ""} - ${nextScheduledCrawlMessage} + ${this.nextScheduledCrawlMessage}
` : ""}
-
- + +
`; } @@ -316,28 +195,321 @@ export class CrawlTemplates extends LiteElement { `; } - private async onSubmit(event: { detail: { formData: FormData } }) { - if (!this.authState) return; + private renderBasicSettings() { + return html` +
+

${msg("Basic settings")}

+
+
+
+ +
+
+
+
+ + (this.scheduleInterval = e.target.value)} + > + ${msg("None")} + ${msg("Daily")} + ${msg("Weekly")} + ${msg("Monthly")} + +
+
+ ${msg("at")} + + (this.scheduleTime = { + ...this.scheduleTime, + hour: +e.target.value, + })} + > + ${hours.map( + ({ value, label }) => + html`${label}` + )} + + : + + (this.scheduleTime = { + ...this.scheduleTime, + minute: +e.target.value, + })} + > + ${minutes.map( + ({ value, label }) => + html`${label}` + )} + + + (this.scheduleTime = { + ...this.scheduleTime, + period: e.target.value, + })} + > + ${msg("AM", { desc: "Time AM/PM" })} + ${msg("PM", { desc: "Time AM/PM" })} + + ${this.timeZoneShortName} +
+
+
+ ${this.nextScheduledCrawlMessage || msg("No crawls scheduled")} +
+
- const { formData } = event.detail; +
+ (this.isRunNow = e.target.checked)} + >${msg("Run immediately on save")} +
+
+ + ${msg("minutes")} + +
+
+ `; + } + + private renderPagesSettings() { + return html` +
+

${msg("Crawl configuration")}

+
+
+
+

+ ${this.isSeedsJsonView + ? msg("Custom Config") + : msg("Configure Seeds")} +

+ (this.isSeedsJsonView = e.target.checked)} + > + ${msg("Use JSON Editor")} + +
+ + ${this.isSeedsJsonView + ? this.renderSeedsJson() + : this.renderSeedsForm()} +
+ `; + } + + private renderSeedsForm() { + return html` + + + Page + Page SPA + Prefix + Host + Any + + + ${msg("pages")} + + `; + } + + private renderSeedsJson() { + return html` +
+
+

+ ${msg( + html`See + Browsertrix Crawler docs + + for all configuration options.` + )} +

+
+ +
+
+ ${this.renderSeedsJsonInput()} + +
+ +
+
+ +
+ ${this.invalidSeedsJsonMessage + ? html` + ${this.invalidSeedsJsonMessage} + ` + : html` ${msg("Valid JSON")} `} +
+
+
+ `; + } + + private renderSeedsJsonInput() { + return html` + + `; + } + + private updateSeedsJson(e: any) { + const textarea = e.target; + const text = textarea.value; + + try { + const json = JSON.parse(text); + + this.seedsJson = JSON.stringify(json, null, 2); + this.invalidSeedsJsonMessage = ""; + + textarea.setCustomValidity(""); + textarea.reportValidity(); + } catch (e: any) { + this.invalidSeedsJsonMessage = e.message + ? msg(str`JSON is invalid: ${e.message.replace("JSON.parse: ", "")}`) + : msg("JSON is invalid."); + } + } + + private parseTemplate(formData: FormData) { const crawlTimeoutMinutes = formData.get("crawlTimeoutMinutes"); const pageLimit = formData.get("limit"); const seedUrlsStr = formData.get("seedUrls"); - const params = { - name: formData.get("name"), + const template: Partial = { + name: formData.get("name") as string, schedule: this.getUTCSchedule(), runNow: this.isRunNow, crawlTimeout: crawlTimeoutMinutes ? +crawlTimeoutMinutes * 60 : 0, - config: { - seeds: (seedUrlsStr as string).trim().replace(/,/g, " ").split(/\s+/g), - scopeType: formData.get("scopeType"), - limit: pageLimit ? +pageLimit : 0, - }, }; + if (this.isSeedsJsonView) { + template.config = JSON.parse(this.seedsJson); + } else { + template.config = { + seeds: (seedUrlsStr as string).trim().replace(/,/g, " ").split(/\s+/g), + scopeType: formData.get("scopeType") as string, + limit: pageLimit ? +pageLimit : 0, + }; + } + + return template; + } + + private async onSubmit(event: { + detail: { formData: FormData }; + target: any; + }) { + if (!this.authState) return; + + if (this.isSeedsJsonView && this.invalidSeedsJsonMessage) { + // Check JSON validity + const jsonEditor = event.target.querySelector("#json-editor"); + + jsonEditor.setCustomValidity(msg("Please correct JSON errors.")); + jsonEditor.reportValidity(); + + return; + } + + const params = this.parseTemplate(event.detail.formData); + console.log(params); + this.serverError = undefined; + this.isSubmitting = true; + try { await this.apiFetch( `/archives/${this.archiveId}/crawlconfigs/`, @@ -348,12 +520,16 @@ export class CrawlTemplates extends LiteElement { } ); - console.debug("success"); - this.navTo(`/archives/${this.archiveId}/crawl-templates`); - } catch (e) { - console.error(e); + } catch (e: any) { + if (e?.isApiError) { + this.serverError = e?.message; + } else { + this.serverError = msg("Something unexpected went wrong"); + } } + + this.isSubmitting = false; } /** diff --git a/frontend/src/shoelace.ts b/frontend/src/shoelace.ts index ef89c96b..83cc1d41 100644 --- a/frontend/src/shoelace.ts +++ b/frontend/src/shoelace.ts @@ -8,6 +8,9 @@ import "@shoelace-style/shoelace/dist/components/alert/alert"; import( /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/button/button" ); +import( + /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/details/details" +); import( /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/dialog/dialog" ); diff --git a/frontend/src/utils/LiteElement.ts b/frontend/src/utils/LiteElement.ts index 10f426b9..e5c0585d 100644 --- a/frontend/src/utils/LiteElement.ts +++ b/frontend/src/utils/LiteElement.ts @@ -55,8 +55,11 @@ export default class LiteElement extends LitElement { if (typeof detail === "string") { errorMessage = detail; } else { - // TODO client error details - errorMessage = "Unknown API error"; + // TODO return client error details + const fieldDetail = detail[0]; + const { loc, msg } = fieldDetail; + + errorMessage = `${loc[loc.length - 1]} ${msg}`; } } catch { errorMessage = "Unknown API error";