From 08f6847194c646471ac86cdbc6d1b53229fed8dd Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Mon, 11 Mar 2024 16:21:20 -0700 Subject: [PATCH] Configurable Max Scale for frontend (#1557) Allow maximum scale option to be fully configurable via `max_crawl_scale`. Already configurable on the backend, and now exposed to the frontend via API `/api/settings` `maxCrawlScale` value. The workflow editor and workflow details are updated to allow selecting the scale up to the maxCrawlScale setting (which defaults to 3 if not set). --- backend/btrixcloud/main.py | 1 + backend/test/test_settings.py | 1 + frontend/src/index.ts | 15 ++++++++++-- frontend/src/pages/org/index.ts | 5 ++++ frontend/src/pages/org/workflow-detail.ts | 30 +++++++++++------------ frontend/src/pages/org/workflow-editor.ts | 18 +++++++++++--- frontend/src/utils/crawler.ts | 2 ++ 7 files changed, 52 insertions(+), 20 deletions(-) diff --git a/backend/btrixcloud/main.py b/backend/btrixcloud/main.py index 4afd8afe..e878fef6 100644 --- a/backend/btrixcloud/main.py +++ b/backend/btrixcloud/main.py @@ -67,6 +67,7 @@ def main(): os.environ.get("DEFAULT_PAGE_LOAD_TIME_SECONDS", 120) ), "maxPagesPerCrawl": int(os.environ.get("MAX_PAGES_PER_CRAWL", 0)), + "maxScale": int(os.environ.get("MAX_CRAWL_SCALE", 3)), } invites = init_invites(mdb, email) diff --git a/backend/test/test_settings.py b/backend/test/test_settings.py index 1f4515ac..a15f012e 100644 --- a/backend/test/test_settings.py +++ b/backend/test/test_settings.py @@ -14,5 +14,6 @@ def test_settings(): "jwtTokenLifetime": 86400, "defaultBehaviorTimeSeconds": 300, "maxPagesPerCrawl": 4, + "maxScale": 3, "defaultPageLoadTimeSeconds": 120, } diff --git a/frontend/src/index.ts b/frontend/src/index.ts index 7afd5ed2..278327ef 100644 --- a/frontend/src/index.ts +++ b/frontend/src/index.ts @@ -33,6 +33,7 @@ import "./assets/fonts/Inter/inter.css"; import "./assets/fonts/Recursive/recursive.css"; import "./styles.css"; import { theme } from "@/theme"; +import { DEFAULT_MAX_SCALE } from "./utils/crawler"; // Make theme CSS available in document document.adoptedStyleSheets = [theme]; @@ -79,6 +80,8 @@ export class App extends LiteElement { @state() private isRegistrationEnabled?: boolean; + private maxScale = DEFAULT_MAX_SCALE; + async connectedCallback() { let authState: AuthState = null; try { @@ -136,6 +139,7 @@ export class App extends LiteElement { if (settings) { this.isRegistrationEnabled = settings.registrationEnabled; + this.maxScale = settings.maxScale; } this.isAppSettingsLoaded = true; @@ -173,13 +177,19 @@ export class App extends LiteElement { } } - async getAppSettings(): Promise<{ registrationEnabled: boolean } | void> { + async getAppSettings(): Promise<{ + registrationEnabled: boolean; + maxScale: number; + } | void> { const resp = await fetch("/api/settings", { headers: { "Content-Type": "application/json" }, }); if (resp.status === 200) { - const body = (await resp.json()) as { registrationEnabled: boolean }; + const body = (await resp.json()) as { + registrationEnabled: boolean; + maxScale: number; + }; return body; } else { @@ -600,6 +610,7 @@ export class App extends LiteElement { .userInfo=${this.appState.userInfo ?? undefined} .viewStateData=${this.viewState.data} .params=${this.viewState.params} + .maxScale=${this.maxScale} slug=${slug} orgPath=${orgPath.split(slug)[1]} orgTab=${orgTab as OrgTab} diff --git a/frontend/src/pages/org/index.ts b/frontend/src/pages/org/index.ts index d0a6cac0..1a0c9b66 100644 --- a/frontend/src/pages/org/index.ts +++ b/frontend/src/pages/org/index.ts @@ -11,6 +11,7 @@ import type { OrgData } from "@/utils/orgs"; import { isAdmin, isCrawler } from "@/utils/orgs"; import LiteElement, { html } from "@/utils/LiteElement"; import { needLogin } from "@/utils/auth"; +import { DEFAULT_MAX_SCALE } from "@/utils/crawler"; import "./workflow-detail"; import "./workflows-list"; import "./workflows-new"; @@ -104,6 +105,9 @@ export class Org extends LiteElement { @property({ type: String }) orgTab: OrgTab = defaultTab; + @property({ type: Number }) + maxScale: number = DEFAULT_MAX_SCALE; + @state() private orgStorageQuotaReached = false; @@ -572,6 +576,7 @@ export class Org extends LiteElement { openDialogName=${this.viewStateData?.dialog} ?isEditing=${isEditing} ?isCrawler=${this.isCrawler} + .maxScale=${this.maxScale} > `; } diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index d047efa7..eaec62f9 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -19,7 +19,11 @@ import type { } from "./types"; import { humanizeSchedule } from "@/utils/cron"; import type { APIPaginatedList } from "@/types/api"; -import { inactiveCrawlStates, isActive } from "@/utils/crawler"; +import { + DEFAULT_MAX_SCALE, + inactiveCrawlStates, + isActive, +} from "@/utils/crawler"; import type { SlSelect } from "@shoelace-style/shoelace"; import type { PageChangeEvent } from "@/components/ui/pagination"; import { ExclusionEditor } from "@/features/crawl-workflows/exclusion-editor"; @@ -70,6 +74,9 @@ export class WorkflowDetail extends LiteElement { @property({ type: String }) initialActivePanel?: Tab; + @property({ type: Number }) + maxScale = DEFAULT_MAX_SCALE; + @state() private workflow?: Workflow; @@ -1261,20 +1268,13 @@ export class WorkflowDetail extends LiteElement { private renderEditScale() { if (!this.workflow) return; - const scaleOptions = [ - { - value: 1, - label: "1×", - }, - { - value: 2, - label: "2×", - }, - { - value: 3, - label: "3×", - }, - ]; + const scaleOptions = []; + for (let value = 1; value <= this.maxScale; value++) { + scaleOptions.push({ + value, + label: `${value}×`, + }); + } return html`
diff --git a/frontend/src/pages/org/workflow-editor.ts b/frontend/src/pages/org/workflow-editor.ts index 87b73fef..72cc2b78 100644 --- a/frontend/src/pages/org/workflow-editor.ts +++ b/frontend/src/pages/org/workflow-editor.ts @@ -17,6 +17,8 @@ import { customElement, } from "lit/decorators.js"; import { when } from "lit/directives/when.js"; +import { map } from "lit/directives/map.js"; +import { range } from "lit/directives/range.js"; import { msg, localized, str } from "@lit/localize"; import { ifDefined } from "lit/directives/if-defined.js"; import { choose } from "lit/directives/choose.js"; @@ -37,6 +39,7 @@ import { getNextDate, } from "@/utils/cron"; import { maxLengthValidator } from "@/utils/form"; +import { DEFAULT_MAX_SCALE } from "@/utils/crawler"; import type { Tab } from "@/components/ui/tab-list"; import type { ExclusionRemoveEvent, @@ -298,6 +301,8 @@ export class CrawlConfigEditor extends LiteElement { @state() private serverError?: TemplateResult | string; + private maxScale = DEFAULT_MAX_SCALE; + // For fuzzy search: private readonly fuse = new Fuse([], { shouldSort: false, @@ -1542,9 +1547,13 @@ https://archiveweb.page/images/${"logo.svg"}`} scale: +(e.target as SlCheckbox).value, })} > - 1× - 2× - 3× + ${map( + range(this.maxScale), + (i: number) => + html` ${i + 1}×`, + )} `)} ${this.renderHelpTextCol( @@ -2562,6 +2571,9 @@ https://archiveweb.page/images/${"logo.svg"}`} if (data.maxPagesPerCrawl > 0) { orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl; } + if (data.maxScale) { + this.maxScale = data.maxScale; + } this.orgDefaults = orgDefaults; } catch (e) { console.debug(e); diff --git a/frontend/src/utils/crawler.ts b/frontend/src/utils/crawler.ts index c2eacda2..4e0c393a 100644 --- a/frontend/src/utils/crawler.ts +++ b/frontend/src/utils/crawler.ts @@ -27,6 +27,8 @@ export const inactiveCrawlStates: CrawlState[] = [ "failed", ]; +export const DEFAULT_MAX_SCALE = 3; + export function isActive(state: CrawlState | null) { return state && activeCrawlStates.includes(state); }