tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Duplicate crawl config from list (#99)

Browse Source
This commit is contained in:
sua yoo 2022-01-25 17:07:54 -08:00 committed by GitHub
parent 3a461d86d4
commit 2666b6f6aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 133 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
frontend/src/index.ts
View File

@ -6,12 +6,12 @@ import type { SlDialog } from "@shoelace-style/shoelace";
import "tailwindcss/tailwind.css";
import type { ArchiveTab } from "./pages/archive";
import type { NotifyEvent } from "./utils/LiteElement";
import type { NotifyEvent, NavigateEvent } from "./utils/LiteElement";
import LiteElement, { html } from "./utils/LiteElement";
import APIRouter from "./utils/APIRouter";
import AuthService from "./utils/AuthService";
import type { LoggedInEvent } from "./utils/AuthService";
import type { ViewState, NavigateEvent } from "./utils/APIRouter";
import type { ViewState } from "./utils/APIRouter";
import type { CurrentUser } from "./types/user";
import type { AuthState } from "./utils/AuthService";
import theme from "./theme";
@ -145,7 +145,7 @@ export class App extends LiteElement {
}
}
navigate(newViewPath: string) {
navigate(newViewPath: string, state?: object) {
if (newViewPath.startsWith("http")) {
const url = new URL(newViewPath);
newViewPath = `${url.pathname}${url.search}`;
@ -158,6 +158,8 @@ export class App extends LiteElement {
this.viewState = this.router.match(newViewPath);
}
this.viewState.data = state;
window.history.pushState(this.viewState, "", this.viewState.pathname);
}
@ -377,6 +379,7 @@ export class App extends LiteElement {
@notify="${this.onNotify}"
.authState=${this.authService.authState}
.userInfo=${this.userInfo}
.viewStateData=${this.viewState.data}
archiveId=${this.viewState.params.id}
archiveTab=${this.viewState.params.crawlConfigId
? "crawl-templates"
@ -472,7 +475,7 @@ export class App extends LiteElement {
onNavigateTo(event: NavigateEvent) {
event.stopPropagation();
this.navigate(event.detail);
this.navigate(event.detail.url, event.detail.state);
}
onUserInfoChange(event: CustomEvent<Partial<CurrentUser>>) {

25
frontend/src/pages/archive/crawl-templates-detail.ts
View File

@ -109,24 +109,24 @@ export class CrawlTemplatesDetail extends LiteElement {
? " border-t"
: ""}"
role="row"
title=${seed.url}
title=${typeof seed === "string" ? seed : seed.url}
>
<div
class="col-span-3 break-all leading-tight"
role="cell"
>
${seed.url}
${typeof seed === "string" ? seed : seed.url}
</div>
<span
class="col-span-1 uppercase text-0-500 text-xs"
role="cell"
>${seed.scopeType ||
>${(typeof seed !== "string" && seed.scopeType) ||
this.crawlTemplate?.config.scopeType}</span
>
<span
class="col-span-1 uppercase text-0-500 text-xs font-mono"
role="cell"
>${seed.limit ||
>${(typeof seed !== "string" && seed.limit) ||
this.crawlTemplate?.config.limit}</span
>
</li>`
@ -290,22 +290,7 @@ export class CrawlTemplatesDetail extends LiteElement {
this.authState!
);
const { config, ...template } = data;
return {
...template,
config: {
...config,
// Normalize seed format
seeds: config.seeds.map((seed) =>
typeof seed === "string"
? {
url: seed,
}
: seed
),
},
};
return data;
}
private async runNow(): Promise<void> {

88
frontend/src/pages/archive/crawl-templates-list.ts
View File

@ -110,9 +110,22 @@ export class CrawlTemplatesList extends LiteElement {
style="font-size: 1rem"
></sl-icon-button>
<ul role="menu">
<ul class="text-sm whitespace-nowrap" role="menu">
<li
class="px-4 py-2 text-danger hover:bg-danger hover:text-white cursor-pointer"
class="p-2 hover:bg-zinc-100 cursor-pointer"
role="menuitem"
@click=${() => this.duplicateConfig(t)}
>
<sl-icon
class="inline-block align-middle px-1"
name="files"
></sl-icon>
<span class="inline-block align-middle pr-2"
>${msg("Duplicate crawl config")}</span
>
</li>
<li
class="p-2 text-danger hover:bg-danger hover:text-white cursor-pointer"
role="menuitem"
@click=${(e: any) => {
// Close dropdown before deleting template
@ -121,7 +134,13 @@ export class CrawlTemplatesList extends LiteElement {
this.deleteTemplate(t);
}}
>
${msg("Delete")}
<sl-icon
class="inline-block align-middle px-1"
name="file-earmark-x"
></sl-icon>
<span class="inline-block align-middle pr-2"
>${msg("Delete")}</span
>
</li>
</ul>
</sl-dropdown>
@ -131,14 +150,20 @@ export class CrawlTemplatesList extends LiteElement {
<div class="grid gap-2 text-xs leading-none">
<div class="overflow-hidden">
<sl-tooltip
content=${t.config.seeds.map(({ url }) => url).join(", ")}
content=${t.config.seeds
.map((seed) =>
typeof seed === "string" ? seed : seed.url
)
.join(", ")}
>
<div
class="font-mono whitespace-nowrap truncate text-0-500"
>
<span class="underline decoration-dashed"
>${t.config.seeds
.map(({ url }) => url)
.map((seed) =>
typeof seed === "string" ? seed : seed.url
)
.join(", ")}</span
>
</div>
@ -251,44 +276,43 @@ export class CrawlTemplatesList extends LiteElement {
* associated with the crawl templates
**/
private async getCrawlTemplates(): Promise<CrawlTemplate[]> {
type CrawlConfig = Omit<CrawlTemplate, "config"> & {
config: Omit<CrawlTemplate["config"], "seeds"> & {
seeds: (string | { url: string })[];
};
};
const data: { crawlConfigs: CrawlConfig[] } = await this.apiFetch(
const data: { crawlConfigs: CrawlTemplate[] } = await this.apiFetch(
`/archives/${this.archiveId}/crawlconfigs`,
this.authState!
);
const crawlConfigs: CrawlTemplate[] = [];
const runningCrawlsMap: RunningCrawlsMap = {};
data.crawlConfigs.forEach(({ config, ...configMeta }) => {
crawlConfigs.push({
...configMeta,
config: {
...config,
// Normalize seed format
seeds: config.seeds.map((seed) =>
typeof seed === "string"
? {
url: seed,
}
: seed
),
},
});
if (configMeta.currCrawlId) {
runningCrawlsMap[configMeta.id] = configMeta.currCrawlId;
data.crawlConfigs.forEach(({ id, currCrawlId }) => {
if (currCrawlId) {
runningCrawlsMap[id] = currCrawlId;
}
});
this.runningCrawlsMap = runningCrawlsMap;
return crawlConfigs;
return data.crawlConfigs;
}
/**
* Create a new template using existing template data
*/
private async duplicateConfig(template: CrawlTemplate) {
const crawlConfig: CrawlTemplate["config"] = {
seeds: template.config.seeds,
scopeType: template.config.scopeType,
limit: template.config.limit,
};
this.navTo(`/archives/${this.archiveId}/crawl-templates/new`, {
crawlConfig,
});
this.notify({
message: msg(str`Copied crawl configuration to new template.`),
type: "success",
icon: "check2-circle",
});
}
private async deleteTemplate(template: CrawlTemplate): Promise<void> {

30
frontend/src/pages/archive/crawl-templates-new.ts
View File

@ -1,4 +1,5 @@
import { state, property } from "lit/decorators.js";
import { ifDefined } from "lit/directives/if-defined.js";
import { msg, localized, str } from "@lit/localize";
import cronParser from "cron-parser";
@ -23,10 +24,8 @@ const initialValues = {
config: {
seeds: [],
scopeType: "prefix",
limit: 0,
},
};
const initialSeedsJson = JSON.stringify(initialValues.config, null, 2);
const hours = Array.from({ length: 12 }).map((x, i) => ({
value: i + 1,
label: `${i + 1}`,
@ -50,6 +49,9 @@ export class CrawlTemplatesNew extends LiteElement {
@property({ type: String })
archiveId!: string;
@property({ type: Object })
initialCrawlConfig?: CrawlConfig;
@state()
private isRunNow: boolean = initialValues.runNow;
@ -69,7 +71,7 @@ export class CrawlTemplatesNew extends LiteElement {
private isSeedsJsonView: boolean = false;
@state()
private seedsJson: string = initialSeedsJson;
private seedsJson: string = "";
@state()
private invalidSeedsJsonMessage: string = "";
@ -111,6 +113,24 @@ export class CrawlTemplatesNew extends LiteElement {
: undefined;
}
connectedCallback(): void {
// Show JSON editor view if complex initial config is specified
// (e.g. cloning a template) since form UI doesn't support
// all available fields in the config
const isComplexConfig = this.initialCrawlConfig?.seeds.some(
(seed: any) => typeof seed !== "string"
);
if (isComplexConfig) {
this.isSeedsJsonView = true;
}
this.initialCrawlConfig = {
...initialValues.config,
...this.initialCrawlConfig,
};
this.seedsJson = JSON.stringify(this.initialCrawlConfig, null, 2);
super.connectedCallback();
}
render() {
return html`
<h2 class="text-xl font-bold mb-3">${msg("New Crawl Template")}</h2>
@ -340,12 +360,13 @@ export class CrawlTemplatesNew extends LiteElement {
"Required. Separate URLs with a new line, space or comma."
)}
rows="3"
value=${this.initialCrawlConfig!.seeds.join("\n")}
required
></sl-textarea>
<sl-select
name="scopeType"
label=${msg("Scope type")}
value=${initialValues.config.scopeType}
value=${this.initialCrawlConfig!.scopeType!}
>
<sl-menu-item value="page">Page</sl-menu-item>
<sl-menu-item value="page-spa">Page SPA</sl-menu-item>
@ -357,6 +378,7 @@ export class CrawlTemplatesNew extends LiteElement {
name="limit"
label=${msg("Page limit")}
type="number"
value=${ifDefined(this.initialCrawlConfig!.limit)}
placeholder=${msg("unlimited")}
>
<span slot="suffix">${msg("pages")}</span>

7
frontend/src/pages/archive/index.ts
View File

@ -1,6 +1,7 @@
import { state, property } from "lit/decorators.js";
import { msg, localized, str } from "@lit/localize";
import type { ViewState } from "../../utils/APIRouter";
import type { AuthState } from "../../utils/AuthService";
import type { CurrentUser } from "../../types/user";
import type { ArchiveData } from "../../utils/archives";
@ -24,6 +25,9 @@ export class Archive extends LiteElement {
@property({ type: Object })
userInfo?: CurrentUser;
@property({ type: Object })
viewStateData?: ViewState["data"];
@property({ type: String })
archiveId?: string;
@ -144,6 +148,8 @@ export class Archive extends LiteElement {
}
private renderCrawlTemplates() {
const crawlConfig = this.viewStateData?.crawlConfig;
if (this.isNewResourceTab || this.crawlConfigId) {
return html`
<div class="md:grid grid-cols-6 gap-6">
@ -171,6 +177,7 @@ export class Archive extends LiteElement {
class="col-span-5 mt-6"
.authState=${this.authState!}
.archiveId=${this.archiveId!}
.initialCrawlConfig=${crawlConfig}
></btrix-crawl-templates-new>`}
</div>
`;

2
frontend/src/pages/archive/types.ts
View File

@ -4,7 +4,7 @@ type SeedConfig = {
};
export type CrawlConfig = {
seeds: ({ url: string } & SeedConfig)[];
seeds: (string | ({ url: string } & SeedConfig))[];
} & SeedConfig;
export type CrawlTemplate = {

3
frontend/src/utils/APIRouter.ts
View File

@ -14,8 +14,9 @@ export type ViewState = {
// e.g. "/users/:id"
// e.g. "/redirect?url"
params: { [key: string]: string };
// arbitrary data to pass between routes
data?: { [key: string]: any };
};
export interface NavigateEvent extends CustomEvent {}
export default class APIRouter {
routes: Routes;

41
frontend/src/utils/LiteElement.ts
View File

@ -3,6 +3,13 @@ import { LitElement, html } from "lit";
import type { Auth } from "../utils/AuthService";
import { APIError } from "./api";
export interface NavigateEvent extends CustomEvent {
detail: {
url: string;
state?: object;
};
}
export interface NotifyEvent extends CustomEvent {
detail: {
/**
@ -27,21 +34,31 @@ export default class LiteElement extends LitElement {
return this;
}
navTo(url: string) {
this.dispatchEvent(
new CustomEvent("navigate", { detail: url, bubbles: true })
);
navTo(url: string, state?: object): void {
const evt: NavigateEvent = new CustomEvent("navigate", {
detail: { url, state },
bubbles: true,
});
this.dispatchEvent(evt);
}
navLink(event: Event) {
/**
* Bind to anchor tag to prevent full page navigation
* @example
* ```ts
* <a href="/" @click=${this.navLink}>go</a>
* ```
* @param event Click event
*/
navLink(event: Event): void {
event.preventDefault();
this.dispatchEvent(
new CustomEvent("navigate", {
detail: (event.currentTarget as HTMLAnchorElement).href,
bubbles: true,
composed: true,
})
);
const evt: NavigateEvent = new CustomEvent("navigate", {
detail: { url: (event.currentTarget as HTMLAnchorElement).href },
bubbles: true,
composed: true,
});
this.dispatchEvent(evt);
}
/**