tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Crawl template enhancements (#114)

Browse Source 
closes #100
This commit is contained in:
sua yoo 2022-01-30 18:30:54 -08:00 committed by GitHub
parent bcbc40059e
commit 7c067ffe36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 176 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
frontend/src/pages/archive/crawl-templates-detail.ts
View File

@ -40,17 +40,6 @@ export class CrawlTemplatesDetail extends LiteElement {
@state() @state()
private editedSchedule?: string; private editedSchedule?: string;
@state()
private isScheduleDisabled?: boolean;
private get timeZone() {
return Intl.DateTimeFormat().resolvedOptions().timeZone;
}
private get timeZoneShortName() {
return getLocaleTimeZone();
}
async firstUpdated() { async firstUpdated() {
try { try {
this.crawlTemplate = await this.getCrawlTemplate(); this.crawlTemplate = await this.getCrawlTemplate();
@ -65,36 +54,44 @@ export class CrawlTemplatesDetail extends LiteElement {
} }
render() { render() {
if (!this.crawlTemplate) { const seeds = this.crawlTemplate?.config.seeds || [];
return html`<div
class="w-full flex items-center justify-center my-24 text-4xl"
>
<sl-spinner></sl-spinner>
</div>`;
}
return html` return html`
<h2 class="text-xl font-bold mb-4">${this.crawlTemplate.name}</h2> <h2 class="text-xl font-bold mb-4 h-7">
${this.crawlTemplate?.name ||
html`<sl-skeleton class="h-7" style="width: 20em"></sl-skeleton>`}
</h2>
${this.crawlTemplate.currCrawlId ${this.renderCurrentlyRunningNotice()}
? html`
<a
class="flex items-center justify-between mb-4 px-3 py-2 border rounded-lg bg-purple-50 border-purple-200 hover:border-purple-500 shadow shadow-purple-200 text-purple-800 transition-colors"
href=${`/archives/${this.archiveId}/crawls/${this.crawlTemplate.currCrawlId}`}
@click=${this.navLink}
>
<span>${msg("View currently running crawl")}</span>
<sl-icon name="arrow-right"></sl-icon>
</a>
`
: ""}
<section class="px-4 py-3 border-t border-b mb-4 text-sm"> <section class="px-4 py-3 border-t border-b mb-4 text-sm">
<dl class="grid grid-cols-2"> <dl class="grid grid-cols-2">
<div>
<dt class="text-xs text-0-600">${msg("Created at")}</dt>
<dd class="h-5">
${this.crawlTemplate?.created
? html`
<sl-format-date
date=${`${this.crawlTemplate.created}Z` /** Z for UTC */}
month="2-digit"
day="2-digit"
year="2-digit"
hour="numeric"
minute="numeric"
time-zone-name="short"
></sl-format-date>
`
: html`<sl-skeleton style="width: 15em"></sl-skeleton>`}
</dd>
</div>
<div> <div>
<dt class="text-xs text-0-600">${msg("Created by")}</dt> <dt class="text-xs text-0-600">${msg("Created by")}</dt>
<!-- TODO show name --> <!-- TODO show name -->
<dd>${this.crawlTemplate.user}</dd> <dd class="h-5">
${this.crawlTemplate?.userName ||
this.crawlTemplate?.userid ||
html`<sl-skeleton style="width: 15em"></sl-skeleton>`}
</dd>
</div> </div>
</dl> </dl>
@ -120,7 +117,7 @@ export class CrawlTemplatesDetail extends LiteElement {
> >
</div> </div>
<ul role="rowgroup"> <ul role="rowgroup">
${this.crawlTemplate.config.seeds ${seeds
.slice(0, this.showAllSeedURLs ? undefined : SEED_URLS_MAX) .slice(0, this.showAllSeedURLs ? undefined : SEED_URLS_MAX)
.map( .map(
(seed, i) => (seed, i) =>
@ -153,7 +150,7 @@ export class CrawlTemplatesDetail extends LiteElement {
)} )}
</ul> </ul>
${this.crawlTemplate.config.seeds.length > SEED_URLS_MAX ${seeds.length > SEED_URLS_MAX
? html`<sl-button ? html`<sl-button
class="mt-2" class="mt-2"
type="neutral" type="neutral"
@ -165,7 +162,7 @@ export class CrawlTemplatesDetail extends LiteElement {
${this.showAllSeedURLs ${this.showAllSeedURLs
? msg("Show less") ? msg("Show less")
: msg(str`Show : msg(str`Show
${this.crawlTemplate.config.seeds.length - SEED_URLS_MAX} ${seeds.length - SEED_URLS_MAX}
more`)} more`)}
</span> </span>
</sl-button>` </sl-button>`
@ -185,7 +182,7 @@ export class CrawlTemplatesDetail extends LiteElement {
<pre <pre
class="language-json bg-gray-800 text-gray-50 p-4 rounded font-mono text-xs" class="language-json bg-gray-800 text-gray-50 p-4 rounded font-mono text-xs"
><code>${JSON.stringify( ><code>${JSON.stringify(
this.crawlTemplate.config, this.crawlTemplate?.config || {},
null, null,
2 2
)}</code></pre> )}</code></pre>
@ -193,7 +190,7 @@ export class CrawlTemplatesDetail extends LiteElement {
<div class="absolute top-2 right-2"> <div class="absolute top-2 right-2">
<btrix-copy-button <btrix-copy-button
.value="${JSON.stringify( .value="${JSON.stringify(
this.crawlTemplate.config, this.crawlTemplate?.config || {},
null, null,
2 2
)}" )}"
@ -217,17 +214,22 @@ export class CrawlTemplatesDetail extends LiteElement {
</div> </div>
<div class="ml-2"> <div class="ml-2">
${this.crawlTemplate
? html`
<sl-button <sl-button
size="small" size="small"
href=${`/archives/${this.archiveId}/crawl-templates/${ href=${`/archives/${this.archiveId}/crawl-templates/${
this.crawlTemplate!.id this.crawlTemplate.id
}${this.isEditing ? "" : "?edit=true"}`} }${this.isEditing ? "" : "?edit=true"}`}
@click=${(e: any) => { @click=${(e: any) => {
const hasChanges = this.isEditing && this.editedSchedule; const hasChanges =
this.isEditing && this.editedSchedule;
if ( if (
!hasChanges || !hasChanges ||
window.confirm( window.confirm(
msg("You have unsaved schedule changes. Are you sure?") msg(
"You have unsaved schedule changes. Are you sure?"
)
) )
) { ) {
this.navLink(e); this.navLink(e);
@ -239,6 +241,8 @@ export class CrawlTemplatesDetail extends LiteElement {
> >
${this.isEditing ? msg("Cancel") : msg("Edit")} ${this.isEditing ? msg("Cancel") : msg("Edit")}
</sl-button> </sl-button>
`
: html`<sl-skeleton></sl-skeleton>`}
</div> </div>
</div> </div>
</div> </div>
@ -253,7 +257,7 @@ export class CrawlTemplatesDetail extends LiteElement {
<div> <div>
<dt class="text-sm text-0-600">${msg("# of Crawls")}</dt> <dt class="text-sm text-0-600">${msg("# of Crawls")}</dt>
<dd class="font-mono"> <dd class="font-mono">
${(this.crawlTemplate.crawlCount || 0).toLocaleString()} ${(this.crawlTemplate?.crawlCount || 0).toLocaleString()}
</dd> </dd>
</div> </div>
<div> <div>
@ -263,6 +267,8 @@ export class CrawlTemplatesDetail extends LiteElement {
<dd <dd
class="flex items-center justify-between border border-zinc-100 rounded p-1 mt-1" class="flex items-center justify-between border border-zinc-100 rounded p-1 mt-1"
> >
${this.crawlTemplate
? html`
${this.crawlTemplate.currCrawlId ${this.crawlTemplate.currCrawlId
? html` <a ? html` <a
class="text-primary font-medium hover:underline text-sm p-1" class="text-primary font-medium hover:underline text-sm p-1"
@ -280,6 +286,8 @@ export class CrawlTemplatesDetail extends LiteElement {
${msg("Run now")} ${msg("Run now")}
</span> </span>
</button>`} </button>`}
`
: html` <sl-skeleton style="width: 6em"></sl-skeleton> `}
</dd> </dd>
</div> </div>
<div> <div>
@ -287,7 +295,7 @@ export class CrawlTemplatesDetail extends LiteElement {
<dd <dd
class="flex items-center justify-between border border-zinc-100 rounded p-1 mt-1" class="flex items-center justify-between border border-zinc-100 rounded p-1 mt-1"
> >
${this.crawlTemplate.lastCrawlId ${this.crawlTemplate?.lastCrawlId
? html`<a ? html`<a
class="text-primary font-medium hover:underline text-sm p-1" class="text-primary font-medium hover:underline text-sm p-1"
href=${`/archives/${this.archiveId}/crawls/${this.crawlTemplate.lastCrawlId}`} href=${`/archives/${this.archiveId}/crawls/${this.crawlTemplate.lastCrawlId}`}
@ -317,24 +325,45 @@ export class CrawlTemplatesDetail extends LiteElement {
`; `;
} }
private renderCurrentlyRunningNotice() {
if (this.crawlTemplate?.currCrawlId) {
return html`
<a
class="flex items-center justify-between mb-4 px-3 py-2 border rounded-lg bg-purple-50 border-purple-200 hover:border-purple-500 shadow shadow-purple-200 text-purple-800 transition-colors"
href=${`/archives/${this.archiveId}/crawls/${this.crawlTemplate.currCrawlId}`}
@click=${this.navLink}
>
<span>${msg("View currently running crawl")}</span>
<sl-icon name="arrow-right"></sl-icon>
</a>
`;
}
return "";
}
private renderReadOnlySchedule() { private renderReadOnlySchedule() {
return html` return html`
<dl class="grid gap-5"> <dl class="grid gap-5">
<div> <div>
<dt class="text-sm text-0-600">${msg("Recurring crawls")}</dt> <dt class="text-sm text-0-600">${msg("Recurring crawls")}</dt>
<dd> <dd>
${this.crawlTemplate!.schedule ${this.crawlTemplate
? html`
${this.crawlTemplate.schedule
? // TODO localize ? // TODO localize
// NOTE human-readable string is in UTC, limitation of library // NOTE human-readable string is in UTC, limitation of library
// currently being used. // currently being used.
// https://github.com/bradymholt/cRonstrue/issues/94 // https://github.com/bradymholt/cRonstrue/issues/94
html`<span html`<span
>${cronstrue.toString(this.crawlTemplate!.schedule, { >${cronstrue.toString(this.crawlTemplate.schedule, {
verbose: true, verbose: true,
})} })}
(in UTC time zone)</span (in UTC time zone)</span
>` >`
: html`<span class="text-0-400">${msg("None")}</span>`} : html`<span class="text-0-400">${msg("None")}</span>`}
`
: html`<sl-skeleton></sl-skeleton>`}
</dd> </dd>
</div> </div>
</dl> </dl>
@ -342,9 +371,13 @@ export class CrawlTemplatesDetail extends LiteElement {
} }
private renderEditSchedule() { private renderEditSchedule() {
if (!this.crawlTemplate) {
return "";
}
return html` return html`
<btrix-crawl-templates-scheduler <btrix-crawl-templates-scheduler
schedule=${this.crawlTemplate!.schedule} schedule=${this.crawlTemplate.schedule}
@submit=${this.onSubmitSchedule} @submit=${this.onSubmitSchedule}
></btrix-crawl-templates-scheduler> ></btrix-crawl-templates-scheduler>
`; `;

34
frontend/src/pages/archive/crawl-templates-list.ts
View File

@ -90,20 +90,29 @@ export class CrawlTemplatesList extends LiteElement {
${this.crawlTemplates.map( ${this.crawlTemplates.map(
(t) => (t) =>
html`<div html`<div
class="col-span-1 p-1 border hover:border-indigo-200 rounded text-sm transition-colors" class="col-span-1 p-1 border shadow hover:shadow-sm hover:bg-zinc-50/50 hover:text-primary rounded text-sm transition-colors"
aria-label=${t.name} aria-label=${t.name}
role="button"
@click=${() => {
this.navTo(
`/archives/${this.archiveId}/crawl-templates/${t.id}`
);
}}
> >
<header class="flex"> <header class="flex">
<a <a
href=${`/archives/${this.archiveId}/crawl-templates/${t.id}`} href=${`/archives/${this.archiveId}/crawl-templates/${t.id}`}
class="block flex-1 px-3 pt-3 font-medium hover:underline whitespace-nowrap truncate mb-1" class="block flex-1 px-3 pt-3 font-medium whitespace-nowrap truncate mb-1"
title=${t.name} title=${t.name}
@click=${this.navLink} @click=${(e: any) => {
e.stopPropagation();
this.navLink(e);
}}
> >
${t.name || "?"} ${t.name || "?"}
</a> </a>
<sl-dropdown> <sl-dropdown @click=${(e: any) => e.stopPropagation()}>
<sl-icon-button <sl-icon-button
slot="trigger" slot="trigger"
name="three-dots-vertical" name="three-dots-vertical"
@ -111,7 +120,7 @@ export class CrawlTemplatesList extends LiteElement {
style="font-size: 1rem" style="font-size: 1rem"
></sl-icon-button> ></sl-icon-button>
<ul class="text-sm whitespace-nowrap" role="menu"> <ul class="text-sm text-0-800 whitespace-nowrap" role="menu">
<li <li
class="p-2 hover:bg-zinc-100 cursor-pointer" class="p-2 hover:bg-zinc-100 cursor-pointer"
role="menuitem" role="menuitem"
@ -165,7 +174,7 @@ export class CrawlTemplatesList extends LiteElement {
</sl-dropdown> </sl-dropdown>
</header> </header>
<div class="px-3 pb-3 flex justify-between items-end"> <div class="px-3 pb-3 flex justify-between items-end text-0-800">
<div class="grid gap-2 text-xs leading-none"> <div class="grid gap-2 text-xs leading-none">
<div class="overflow-hidden"> <div class="overflow-hidden">
<sl-tooltip <sl-tooltip
@ -267,14 +276,16 @@ export class CrawlTemplatesList extends LiteElement {
.runningCrawlsMap[t.id] .runningCrawlsMap[t.id]
? "bg-purple-50" ? "bg-purple-50"
: "bg-white"} border-purple-200 hover:border-purple-500 text-purple-600 transition-colors" : "bg-white"} border-purple-200 hover:border-purple-500 text-purple-600 transition-colors"
@click=${() => @click=${(e: any) => {
e.stopPropagation();
this.runningCrawlsMap[t.id] this.runningCrawlsMap[t.id]
? this.navTo( ? this.navTo(
`/archives/${this.archiveId}/crawls/${ `/archives/${this.archiveId}/crawls/${
this.runningCrawlsMap[t.id] this.runningCrawlsMap[t.id]
}` }`
) )
: this.runNow(t)} : this.runNow(t);
}}
> >
<span class="whitespace-nowrap"> <span class="whitespace-nowrap">
${this.runningCrawlsMap[t.id] ${this.runningCrawlsMap[t.id]
@ -337,14 +348,17 @@ export class CrawlTemplatesList extends LiteElement {
* Create a new template using existing template data * Create a new template using existing template data
*/ */
private async duplicateConfig(template: CrawlTemplate) { private async duplicateConfig(template: CrawlTemplate) {
const crawlConfig: CrawlTemplate["config"] = { const config: CrawlTemplate["config"] = {
seeds: template.config.seeds, seeds: template.config.seeds,
scopeType: template.config.scopeType, scopeType: template.config.scopeType,
limit: template.config.limit, limit: template.config.limit,
}; };
this.navTo(`/archives/${this.archiveId}/crawl-templates/new`, { this.navTo(`/archives/${this.archiveId}/crawl-templates/new`, {
crawlConfig, crawlTemplate: {
name: msg(str`${template.name} Copy`),
config,
},
}); });
this.notify({ this.notify({

36
frontend/src/pages/archive/crawl-templates-new.ts
View File

@ -50,7 +50,10 @@ export class CrawlTemplatesNew extends LiteElement {
archiveId!: string; archiveId!: string;
@property({ type: Object }) @property({ type: Object })
initialCrawlConfig?: CrawlConfig; initialCrawlTemplate?: {
name: string;
config: CrawlConfig;
};
@state() @state()
private isRunNow: boolean = initialValues.runNow; private isRunNow: boolean = initialValues.runNow;
@ -117,17 +120,20 @@ export class CrawlTemplatesNew extends LiteElement {
// Show JSON editor view if complex initial config is specified // Show JSON editor view if complex initial config is specified
// (e.g. cloning a template) since form UI doesn't support // (e.g. cloning a template) since form UI doesn't support
// all available fields in the config // all available fields in the config
const isComplexConfig = this.initialCrawlConfig?.seeds.some( const isComplexConfig = this.initialCrawlTemplate?.config.seeds.some(
(seed: any) => typeof seed !== "string" (seed: any) => typeof seed !== "string"
); );
if (isComplexConfig) { if (isComplexConfig) {
this.isSeedsJsonView = true; this.isSeedsJsonView = true;
} }
this.initialCrawlConfig = { this.initialCrawlTemplate = {
name: this.initialCrawlTemplate?.name || initialValues.name,
config: {
...initialValues.config, ...initialValues.config,
...this.initialCrawlConfig, ...this.initialCrawlTemplate?.config,
},
}; };
this.seedsJson = JSON.stringify(this.initialCrawlConfig, null, 2); this.seedsJson = JSON.stringify(this.initialCrawlTemplate.config, null, 2);
super.connectedCallback(); super.connectedCallback();
} }
@ -209,7 +215,7 @@ export class CrawlTemplatesNew extends LiteElement {
desc: "Example crawl template name", desc: "Example crawl template name",
})} })}
autocomplete="off" autocomplete="off"
value=${initialValues.name} value=${this.initialCrawlTemplate!.name}
required required
></sl-input> ></sl-input>
</section> </section>
@ -300,12 +306,12 @@ export class CrawlTemplatesNew extends LiteElement {
</div> </div>
</div> </div>
<sl-switch <sl-checkbox
name="runNow" name="runNow"
?checked=${initialValues.runNow} ?checked=${initialValues.runNow}
@sl-change=${(e: any) => (this.isRunNow = e.target.checked)} @sl-change=${(e: any) => (this.isRunNow = e.target.checked)}
>${msg("Run immediately on save")}</sl-switch >${msg("Run immediately on save")}
> </sl-checkbox>
<sl-input <sl-input
name="crawlTimeoutMinutes" name="crawlTimeoutMinutes"
@ -358,13 +364,13 @@ export class CrawlTemplatesNew extends LiteElement {
"Required. Separate URLs with a new line, space or comma." "Required. Separate URLs with a new line, space or comma."
)} )}
rows="3" rows="3"
value=${this.initialCrawlConfig!.seeds.join("\n")} value=${this.initialCrawlTemplate!.config.seeds.join("\n")}
required required
></sl-textarea> ></sl-textarea>
<sl-select <sl-select
name="scopeType" name="scopeType"
label=${msg("Crawl Scope")} label=${msg("Crawl Scope")}
value=${this.initialCrawlConfig!.scopeType!} value=${this.initialCrawlTemplate!.config.scopeType!}
> >
<sl-menu-item value="page">Page</sl-menu-item> <sl-menu-item value="page">Page</sl-menu-item>
<sl-menu-item value="page-spa">Page SPA</sl-menu-item> <sl-menu-item value="page-spa">Page SPA</sl-menu-item>
@ -379,7 +385,7 @@ export class CrawlTemplatesNew extends LiteElement {
name="limit" name="limit"
label=${msg("Page Limit")} label=${msg("Page Limit")}
type="number" type="number"
value=${ifDefined(this.initialCrawlConfig!.limit)} value=${ifDefined(this.initialCrawlTemplate!.config.limit)}
placeholder=${msg("unlimited")} placeholder=${msg("unlimited")}
> >
<span slot="suffix">${msg("pages")}</span> <span slot="suffix">${msg("pages")}</span>
@ -491,11 +497,7 @@ export class CrawlTemplatesNew extends LiteElement {
template.config = JSON.parse(this.seedsJson); template.config = JSON.parse(this.seedsJson);
} else { } else {
template.config = { template.config = {
seeds: (seedUrlsStr as string) seeds: (seedUrlsStr as string).trim().replace(/,/g, " ").split(/\s+/g),
.trim()
.replace(/,/g, " ")
.split(/\s+/g)
.map((url) => ({ url })),
scopeType: formData.get("scopeType") as string, scopeType: formData.get("scopeType") as string,
limit: pageLimit ? +pageLimit : 0, limit: pageLimit ? +pageLimit : 0,
extraHops: formData.get("extraHopsOne") ? 1 : 0, extraHops: formData.get("extraHopsOne") ? 1 : 0,

4
frontend/src/pages/archive/index.ts
View File

@ -172,13 +172,13 @@ export class Archive extends LiteElement {
} }
if (this.isNewResourceTab) { if (this.isNewResourceTab) {
const crawlConfig = this.viewStateData?.crawlConfig; const crawlTemplate = this.viewStateData?.crawlTemplate;
return html` <btrix-crawl-templates-new return html` <btrix-crawl-templates-new
class="col-span-5 mt-6" class="col-span-5 mt-6"
.authState=${this.authState!} .authState=${this.authState!}
.archiveId=${this.archiveId!} .archiveId=${this.archiveId!}
.initialCrawlConfig=${crawlConfig} .initialCrawlTemplate=${crawlTemplate}
></btrix-crawl-templates-new>`; ></btrix-crawl-templates-new>`;
} }

4
frontend/src/pages/archive/types.ts
View File

@ -12,7 +12,9 @@ export type CrawlTemplate = {
id: string; id: string;
name: string; name: string;
schedule: string; schedule: string;
user: string; userid: string;
userName?: string;
created: string;
crawlCount: number; crawlCount: number;
lastCrawlId: string; lastCrawlId: string;
lastCrawlTime: string; lastCrawlTime: string;

3
frontend/src/shoelace.ts
View File

@ -50,6 +50,9 @@ import(
import( import(
/* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/select/select" /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/select/select"
); );
import(
/* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/skeleton/skeleton"
);
import( import(
/* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/spinner/spinner" /* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/spinner/spinner"
); );