Configurable Max Scale for frontend (#1557)
Allow maximum scale option to be fully configurable via `max_crawl_scale`. Already configurable on the backend, and now exposed to the frontend via API `/api/settings` `maxCrawlScale` value. The workflow editor and workflow details are updated to allow selecting the scale up to the maxCrawlScale setting (which defaults to 3 if not set).
This commit is contained in:
parent
8462c08206
commit
08f6847194
@ -67,6 +67,7 @@ def main():
|
||||
os.environ.get("DEFAULT_PAGE_LOAD_TIME_SECONDS", 120)
|
||||
),
|
||||
"maxPagesPerCrawl": int(os.environ.get("MAX_PAGES_PER_CRAWL", 0)),
|
||||
"maxScale": int(os.environ.get("MAX_CRAWL_SCALE", 3)),
|
||||
}
|
||||
|
||||
invites = init_invites(mdb, email)
|
||||
|
@ -14,5 +14,6 @@ def test_settings():
|
||||
"jwtTokenLifetime": 86400,
|
||||
"defaultBehaviorTimeSeconds": 300,
|
||||
"maxPagesPerCrawl": 4,
|
||||
"maxScale": 3,
|
||||
"defaultPageLoadTimeSeconds": 120,
|
||||
}
|
||||
|
@ -33,6 +33,7 @@ import "./assets/fonts/Inter/inter.css";
|
||||
import "./assets/fonts/Recursive/recursive.css";
|
||||
import "./styles.css";
|
||||
import { theme } from "@/theme";
|
||||
import { DEFAULT_MAX_SCALE } from "./utils/crawler";
|
||||
|
||||
// Make theme CSS available in document
|
||||
document.adoptedStyleSheets = [theme];
|
||||
@ -79,6 +80,8 @@ export class App extends LiteElement {
|
||||
@state()
|
||||
private isRegistrationEnabled?: boolean;
|
||||
|
||||
private maxScale = DEFAULT_MAX_SCALE;
|
||||
|
||||
async connectedCallback() {
|
||||
let authState: AuthState = null;
|
||||
try {
|
||||
@ -136,6 +139,7 @@ export class App extends LiteElement {
|
||||
|
||||
if (settings) {
|
||||
this.isRegistrationEnabled = settings.registrationEnabled;
|
||||
this.maxScale = settings.maxScale;
|
||||
}
|
||||
|
||||
this.isAppSettingsLoaded = true;
|
||||
@ -173,13 +177,19 @@ export class App extends LiteElement {
|
||||
}
|
||||
}
|
||||
|
||||
async getAppSettings(): Promise<{ registrationEnabled: boolean } | void> {
|
||||
async getAppSettings(): Promise<{
|
||||
registrationEnabled: boolean;
|
||||
maxScale: number;
|
||||
} | void> {
|
||||
const resp = await fetch("/api/settings", {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
|
||||
if (resp.status === 200) {
|
||||
const body = (await resp.json()) as { registrationEnabled: boolean };
|
||||
const body = (await resp.json()) as {
|
||||
registrationEnabled: boolean;
|
||||
maxScale: number;
|
||||
};
|
||||
|
||||
return body;
|
||||
} else {
|
||||
@ -600,6 +610,7 @@ export class App extends LiteElement {
|
||||
.userInfo=${this.appState.userInfo ?? undefined}
|
||||
.viewStateData=${this.viewState.data}
|
||||
.params=${this.viewState.params}
|
||||
.maxScale=${this.maxScale}
|
||||
slug=${slug}
|
||||
orgPath=${orgPath.split(slug)[1]}
|
||||
orgTab=${orgTab as OrgTab}
|
||||
|
@ -11,6 +11,7 @@ import type { OrgData } from "@/utils/orgs";
|
||||
import { isAdmin, isCrawler } from "@/utils/orgs";
|
||||
import LiteElement, { html } from "@/utils/LiteElement";
|
||||
import { needLogin } from "@/utils/auth";
|
||||
import { DEFAULT_MAX_SCALE } from "@/utils/crawler";
|
||||
import "./workflow-detail";
|
||||
import "./workflows-list";
|
||||
import "./workflows-new";
|
||||
@ -104,6 +105,9 @@ export class Org extends LiteElement {
|
||||
@property({ type: String })
|
||||
orgTab: OrgTab = defaultTab;
|
||||
|
||||
@property({ type: Number })
|
||||
maxScale: number = DEFAULT_MAX_SCALE;
|
||||
|
||||
@state()
|
||||
private orgStorageQuotaReached = false;
|
||||
|
||||
@ -572,6 +576,7 @@ export class Org extends LiteElement {
|
||||
openDialogName=${this.viewStateData?.dialog}
|
||||
?isEditing=${isEditing}
|
||||
?isCrawler=${this.isCrawler}
|
||||
.maxScale=${this.maxScale}
|
||||
></btrix-workflow-detail>
|
||||
`;
|
||||
}
|
||||
|
@ -19,7 +19,11 @@ import type {
|
||||
} from "./types";
|
||||
import { humanizeSchedule } from "@/utils/cron";
|
||||
import type { APIPaginatedList } from "@/types/api";
|
||||
import { inactiveCrawlStates, isActive } from "@/utils/crawler";
|
||||
import {
|
||||
DEFAULT_MAX_SCALE,
|
||||
inactiveCrawlStates,
|
||||
isActive,
|
||||
} from "@/utils/crawler";
|
||||
import type { SlSelect } from "@shoelace-style/shoelace";
|
||||
import type { PageChangeEvent } from "@/components/ui/pagination";
|
||||
import { ExclusionEditor } from "@/features/crawl-workflows/exclusion-editor";
|
||||
@ -70,6 +74,9 @@ export class WorkflowDetail extends LiteElement {
|
||||
@property({ type: String })
|
||||
initialActivePanel?: Tab;
|
||||
|
||||
@property({ type: Number })
|
||||
maxScale = DEFAULT_MAX_SCALE;
|
||||
|
||||
@state()
|
||||
private workflow?: Workflow;
|
||||
|
||||
@ -1261,20 +1268,13 @@ export class WorkflowDetail extends LiteElement {
|
||||
private renderEditScale() {
|
||||
if (!this.workflow) return;
|
||||
|
||||
const scaleOptions = [
|
||||
{
|
||||
value: 1,
|
||||
label: "1×",
|
||||
},
|
||||
{
|
||||
value: 2,
|
||||
label: "2×",
|
||||
},
|
||||
{
|
||||
value: 3,
|
||||
label: "3×",
|
||||
},
|
||||
];
|
||||
const scaleOptions = [];
|
||||
for (let value = 1; value <= this.maxScale; value++) {
|
||||
scaleOptions.push({
|
||||
value,
|
||||
label: `${value}×`,
|
||||
});
|
||||
}
|
||||
|
||||
return html`
|
||||
<div>
|
||||
|
@ -17,6 +17,8 @@ import {
|
||||
customElement,
|
||||
} from "lit/decorators.js";
|
||||
import { when } from "lit/directives/when.js";
|
||||
import { map } from "lit/directives/map.js";
|
||||
import { range } from "lit/directives/range.js";
|
||||
import { msg, localized, str } from "@lit/localize";
|
||||
import { ifDefined } from "lit/directives/if-defined.js";
|
||||
import { choose } from "lit/directives/choose.js";
|
||||
@ -37,6 +39,7 @@ import {
|
||||
getNextDate,
|
||||
} from "@/utils/cron";
|
||||
import { maxLengthValidator } from "@/utils/form";
|
||||
import { DEFAULT_MAX_SCALE } from "@/utils/crawler";
|
||||
import type { Tab } from "@/components/ui/tab-list";
|
||||
import type {
|
||||
ExclusionRemoveEvent,
|
||||
@ -298,6 +301,8 @@ export class CrawlConfigEditor extends LiteElement {
|
||||
@state()
|
||||
private serverError?: TemplateResult | string;
|
||||
|
||||
private maxScale = DEFAULT_MAX_SCALE;
|
||||
|
||||
// For fuzzy search:
|
||||
private readonly fuse = new Fuse<string>([], {
|
||||
shouldSort: false,
|
||||
@ -1542,9 +1547,13 @@ https://archiveweb.page/images/${"logo.svg"}`}
|
||||
scale: +(e.target as SlCheckbox).value,
|
||||
})}
|
||||
>
|
||||
<sl-radio-button value="1" size="small">1×</sl-radio-button>
|
||||
<sl-radio-button value="2" size="small">2×</sl-radio-button>
|
||||
<sl-radio-button value="3" size="small">3×</sl-radio-button>
|
||||
${map(
|
||||
range(this.maxScale),
|
||||
(i: number) =>
|
||||
html` <sl-radio-button value="${i + 1}" size="small"
|
||||
>${i + 1}×</sl-radio-button
|
||||
>`,
|
||||
)}
|
||||
</sl-radio-group>
|
||||
`)}
|
||||
${this.renderHelpTextCol(
|
||||
@ -2562,6 +2571,9 @@ https://archiveweb.page/images/${"logo.svg"}`}
|
||||
if (data.maxPagesPerCrawl > 0) {
|
||||
orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl;
|
||||
}
|
||||
if (data.maxScale) {
|
||||
this.maxScale = data.maxScale;
|
||||
}
|
||||
this.orgDefaults = orgDefaults;
|
||||
} catch (e) {
|
||||
console.debug(e);
|
||||
|
@ -27,6 +27,8 @@ export const inactiveCrawlStates: CrawlState[] = [
|
||||
"failed",
|
||||
];
|
||||
|
||||
export const DEFAULT_MAX_SCALE = 3;
|
||||
|
||||
export function isActive(state: CrawlState | null) {
|
||||
return state && activeCrawlStates.includes(state);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user