Configurable Max Scale for frontend (#1557)

Allow maximum scale option to be fully configurable via
`max_crawl_scale`. Already configurable on the backend, and now exposed
to the frontend via API `/api/settings` `maxCrawlScale` value.

The workflow editor and workflow details are updated to allow selecting
the scale up to the maxCrawlScale setting (which defaults to 3 if not
set).

backend/btrixcloud/main.py

@@ -67,6 +67,7 @@ def main():
             os.environ.get("DEFAULT_PAGE_LOAD_TIME_SECONDS", 120)
         ),
         "maxPagesPerCrawl": int(os.environ.get("MAX_PAGES_PER_CRAWL", 0)),
+        "maxScale": int(os.environ.get("MAX_CRAWL_SCALE", 3)),
     }
 
     invites = init_invites(mdb, email)

backend/test/test_settings.py

@@ -14,5 +14,6 @@ def test_settings():
         "jwtTokenLifetime": 86400,
         "defaultBehaviorTimeSeconds": 300,
         "maxPagesPerCrawl": 4,
+        "maxScale": 3,
         "defaultPageLoadTimeSeconds": 120,
     }

frontend/src/index.ts

@@ -33,6 +33,7 @@ import "./assets/fonts/Inter/inter.css";
 import "./assets/fonts/Recursive/recursive.css";
 import "./styles.css";
 import { theme } from "@/theme";
+import { DEFAULT_MAX_SCALE } from "./utils/crawler";
 
 // Make theme CSS available in document
 document.adoptedStyleSheets = [theme];
@@ -79,6 +80,8 @@ export class App extends LiteElement {
   @state()
   private isRegistrationEnabled?: boolean;
 
+  private maxScale = DEFAULT_MAX_SCALE;
+
   async connectedCallback() {
     let authState: AuthState = null;
     try {
@@ -136,6 +139,7 @@ export class App extends LiteElement {
 
     if (settings) {
       this.isRegistrationEnabled = settings.registrationEnabled;
+      this.maxScale = settings.maxScale;
     }
 
     this.isAppSettingsLoaded = true;
@@ -173,13 +177,19 @@ export class App extends LiteElement {
     }
   }
 
-  async getAppSettings(): Promise<{ registrationEnabled: boolean } | void> {
+  async getAppSettings(): Promise<{
+    registrationEnabled: boolean;
+    maxScale: number;
+  } | void> {
     const resp = await fetch("/api/settings", {
       headers: { "Content-Type": "application/json" },
     });
 
     if (resp.status === 200) {
-      const body = (await resp.json()) as { registrationEnabled: boolean };
+      const body = (await resp.json()) as {
+        registrationEnabled: boolean;
+        maxScale: number;
+      };
 
       return body;
     } else {
@@ -600,6 +610,7 @@ export class App extends LiteElement {
           .userInfo=${this.appState.userInfo ?? undefined}
           .viewStateData=${this.viewState.data}
           .params=${this.viewState.params}
+          .maxScale=${this.maxScale}
           slug=${slug}
           orgPath=${orgPath.split(slug)[1]}
           orgTab=${orgTab as OrgTab}

frontend/src/pages/org/index.ts

@@ -11,6 +11,7 @@ import type { OrgData } from "@/utils/orgs";
 import { isAdmin, isCrawler } from "@/utils/orgs";
 import LiteElement, { html } from "@/utils/LiteElement";
 import { needLogin } from "@/utils/auth";
+import { DEFAULT_MAX_SCALE } from "@/utils/crawler";
 import "./workflow-detail";
 import "./workflows-list";
 import "./workflows-new";
@@ -104,6 +105,9 @@ export class Org extends LiteElement {
   @property({ type: String })
   orgTab: OrgTab = defaultTab;
 
+  @property({ type: Number })
+  maxScale: number = DEFAULT_MAX_SCALE;
+
   @state()
   private orgStorageQuotaReached = false;
 
@@ -572,6 +576,7 @@ export class Org extends LiteElement {
           openDialogName=${this.viewStateData?.dialog}
           ?isEditing=${isEditing}
           ?isCrawler=${this.isCrawler}
+          .maxScale=${this.maxScale}
         ></btrix-workflow-detail>
       `;
     }

frontend/src/pages/org/workflow-detail.ts

@@ -19,7 +19,11 @@ import type {
 } from "./types";
 import { humanizeSchedule } from "@/utils/cron";
 import type { APIPaginatedList } from "@/types/api";
-import { inactiveCrawlStates, isActive } from "@/utils/crawler";
+import {
+  DEFAULT_MAX_SCALE,
+  inactiveCrawlStates,
+  isActive,
+} from "@/utils/crawler";
 import type { SlSelect } from "@shoelace-style/shoelace";
 import type { PageChangeEvent } from "@/components/ui/pagination";
 import { ExclusionEditor } from "@/features/crawl-workflows/exclusion-editor";
@@ -70,6 +74,9 @@ export class WorkflowDetail extends LiteElement {
   @property({ type: String })
   initialActivePanel?: Tab;
 
+  @property({ type: Number })
+  maxScale = DEFAULT_MAX_SCALE;
+
   @state()
   private workflow?: Workflow;
 
@@ -1261,20 +1268,13 @@ export class WorkflowDetail extends LiteElement {
   private renderEditScale() {
     if (!this.workflow) return;
 
-    const scaleOptions = [
+    const scaleOptions = [];
-      {
+    for (let value = 1; value <= this.maxScale; value++) {
-        value: 1,
+      scaleOptions.push({
-        label: "1×",
+        value,
-      },
+        label: `${value}×`,
-      {
+      });
-        value: 2,
+    }
-        label: "2×",
-      },
-      {
-        value: 3,
-        label: "3×",
-      },
-    ];
 
     return html`
       <div>

frontend/src/pages/org/workflow-editor.ts

@@ -17,6 +17,8 @@ import {
   customElement,
 } from "lit/decorators.js";
 import { when } from "lit/directives/when.js";
+import { map } from "lit/directives/map.js";
+import { range } from "lit/directives/range.js";
 import { msg, localized, str } from "@lit/localize";
 import { ifDefined } from "lit/directives/if-defined.js";
 import { choose } from "lit/directives/choose.js";
@@ -37,6 +39,7 @@ import {
   getNextDate,
 } from "@/utils/cron";
 import { maxLengthValidator } from "@/utils/form";
+import { DEFAULT_MAX_SCALE } from "@/utils/crawler";
 import type { Tab } from "@/components/ui/tab-list";
 import type {
   ExclusionRemoveEvent,
@@ -298,6 +301,8 @@ export class CrawlConfigEditor extends LiteElement {
   @state()
   private serverError?: TemplateResult | string;
 
+  private maxScale = DEFAULT_MAX_SCALE;
+
   // For fuzzy search:
   private readonly fuse = new Fuse<string>([], {
     shouldSort: false,
@@ -1542,9 +1547,13 @@ https://archiveweb.page/images/${"logo.svg"}`}
               scale: +(e.target as SlCheckbox).value,
             })}
         >
-          <sl-radio-button value="1" size="small">1×</sl-radio-button>
+          ${map(
-          <sl-radio-button value="2" size="small">2×</sl-radio-button>
+            range(this.maxScale),
-          <sl-radio-button value="3" size="small">3×</sl-radio-button>
+            (i: number) =>
+              html` <sl-radio-button value="${i + 1}" size="small"
+                >${i + 1}×</sl-radio-button
+              >`,
+          )}
         </sl-radio-group>
       `)}
       ${this.renderHelpTextCol(
@@ -2562,6 +2571,9 @@ https://archiveweb.page/images/${"logo.svg"}`}
       if (data.maxPagesPerCrawl > 0) {
         orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl;
       }
+      if (data.maxScale) {
+        this.maxScale = data.maxScale;
+      }
       this.orgDefaults = orgDefaults;
     } catch (e) {
       console.debug(e);

frontend/src/utils/crawler.ts

@@ -27,6 +27,8 @@ export const inactiveCrawlStates: CrawlState[] = [
   "failed",
 ];
 
+export const DEFAULT_MAX_SCALE = 3;
+
 export function isActive(state: CrawlState | null) {
   return state && activeCrawlStates.includes(state);
 }