Set max pages to API default (#739)

frontend/src/components/config-details.ts

@@ -29,6 +29,12 @@ export class ConfigDetails extends LiteElement {
   @property({ type: Boolean })
   hideTags = false;
 
+  @state()
+  private orgDefaults = {
+    behaviorTimeoutMinutes: Infinity,
+    maxPagesPerCrawl: Infinity,
+  };
+
   private readonly scopeTypeLabels: Record<
     CrawlConfig["config"]["scopeType"],
     string
@@ -42,6 +48,11 @@ export class ConfigDetails extends LiteElement {
     any: msg("Any"),
   };
 
+  connectedCallback() {
+    super.connectedCallback();
+    this.fetchAPIDefaults();
+  }
+
   render() {
     const crawlConfig = this.crawlConfig;
     const exclusions = crawlConfig?.config.exclude || [];
@@ -204,6 +215,7 @@ export class ConfigDetails extends LiteElement {
     let primarySeedConfig: SeedConfig | Seed = seedsConfig;
     let primarySeedUrl = seedsConfig.seeds[0].url;
     const includeUrlList = primarySeedConfig.include || seedsConfig.include;
+    const maxPages = primarySeedConfig.limit ?? seedsConfig.limit;
     return html`
       ${this.renderSetting(msg("Primary Seed URL"), primarySeedUrl, true)}
       ${this.renderSetting(
@@ -245,9 +257,16 @@ export class ConfigDetails extends LiteElement {
       )}
       ${this.renderSetting(
         msg("Max Pages"),
-        seedsConfig.limit
+        when(
-          ? msg(str`${primarySeedConfig.limit ?? seedsConfig.limit} pages`)
+          maxPages,
-          : msg("Unlimited")
+          () => msg(str`${maxPages} page(s)`),
+          () =>
+            this.orgDefaults.maxPagesPerCrawl < Infinity
+              ? msg(
+                  str`Maximum Allowed (${this.orgDefaults.maxPagesPerCrawl.toLocaleString()} pages)`
+                )
+              : undefined
+        )
       )}
     `;
   };
@@ -283,4 +302,27 @@ export class ConfigDetails extends LiteElement {
       </btrix-desc-list-item>
     `;
   }
+
+  private async fetchAPIDefaults() {
+    const orgDefaults = { ...this.orgDefaults };
+    try {
+      const resp = await fetch("/api/settings", {
+        headers: { "Content-Type": "application/json" },
+      });
+      if (!resp.ok) {
+        throw new Error(resp.statusText);
+      }
+      const data = await resp.json();
+      if (data.defaultBehaviorTimeSeconds) {
+        orgDefaults.behaviorTimeoutMinutes =
+          data.defaultBehaviorTimeSeconds / 60;
+      }
+      if (data.maxPagesPerCrawl > 0) {
+        orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl;
+      }
+    } catch (e: any) {
+      console.debug(e);
+    }
+    this.orgDefaults = orgDefaults;
+  }
 }

frontend/src/pages/org/workflow-editor.ts

@@ -192,6 +192,7 @@ const urlListToArray = flow(
   trimArray
 );
 const DEFAULT_BEHAVIOR_TIMEOUT_MINUTES = 5;
+const DEFAULT_MAX_PAGES_PER_CRAWL = Infinity;
 
 @localized()
 export class CrawlConfigEditor extends LiteElement {
@@ -220,7 +221,10 @@ export class CrawlConfigEditor extends LiteElement {
   private progressState!: ProgressState;
 
   @state()
-  private defaultBehaviorTimeoutMinutes?: number;
+  private orgDefaults = {
+    behaviorTimeoutMinutes: DEFAULT_BEHAVIOR_TIMEOUT_MINUTES,
+    maxPagesPerCrawl: DEFAULT_MAX_PAGES_PER_CRAWL,
+  };
 
   @state()
   private formState!: FormState;
@@ -1172,14 +1176,37 @@ https://archiveweb.page/images/${"logo.svg"}`}
             type="number"
             value=${this.formState.pageLimit || ""}
             min=${minPages}
-            placeholder=${msg("Unlimited")}
+            max=${this.orgDefaults.maxPagesPerCrawl}
+            placeholder=${this.orgDefaults.maxPagesPerCrawl === Infinity
+              ? msg("Unlimited")
+              : msg(
+                  str`Maximum Allowed (${this.orgDefaults.maxPagesPerCrawl.toLocaleString()})`
+                )}
+            @sl-input=${async (e: CustomEvent) => {
+              const inputEl = e.target as SlInput;
+              await inputEl.updateComplete;
+              let helpText = "";
+              if (inputEl.invalid) {
+                const value = +inputEl.value;
+                if (value < minPages) {
+                  helpText =
+                    minPages === 1
+                      ? msg(
+                          str`Minimum ${minPages.toLocaleString()} page per crawl`
+                        )
+                      : msg(
+                          str`Minimum ${minPages.toLocaleString()} pages per crawl`
+                        );
+                } else if (value > this.orgDefaults.maxPagesPerCrawl) {
+                  helpText = msg(
+                    str`Maximum ${this.orgDefaults.maxPagesPerCrawl.toLocaleString()} pages per crawl`
+                  );
+                }
+              }
+              inputEl.helpText = helpText;
+            }}
           >
             <span slot="suffix">${msg("pages")}</span>
-            <div slot="help-text">
-              ${minPages === 1
-                ? msg(str`Minimum ${minPages} page`)
-                : msg(str`Minimum ${minPages} pages`)}
-            </div>
           </sl-input>
         </sl-mutation-observer>
       `)}
@@ -1195,9 +1222,9 @@ https://archiveweb.page/images/${"logo.svg"}`}
           placeholder=${msg("Unlimited")}
           value=${ifDefined(
             this.formState.pageTimeoutMinutes ??
-              this.defaultBehaviorTimeoutMinutes
+              this.orgDefaults.behaviorTimeoutMinutes
           )}
-          ?disabled=${this.defaultBehaviorTimeoutMinutes === undefined}
+          ?disabled=${this.orgDefaults.behaviorTimeoutMinutes === undefined}
           min="1"
           required
         >
@@ -1928,7 +1955,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
           : this.parseUrlListConfig()),
         behaviorTimeout:
           (this.formState.pageTimeoutMinutes ??
-            this.defaultBehaviorTimeoutMinutes ??
+            this.orgDefaults.behaviorTimeoutMinutes ??
             DEFAULT_BEHAVIOR_TIMEOUT_MINUTES) * 60,
         limit: this.formState.pageLimit ? +this.formState.pageLimit : undefined,
         lang: this.formState.lang || "",
@@ -1943,7 +1970,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
   private parseUrlListConfig(): NewCrawlConfigParams["config"] {
     const config = {
       seeds: urlListToArray(this.formState.urlList).map((seedUrl) => {
-        const newSeed: Seed = {url: seedUrl, scopeType: "page"};
+        const newSeed: Seed = { url: seedUrl, scopeType: "page" };
         return newSeed;
       }),
       scopeType: "page" as FormState["scopeType"],
@@ -1960,9 +1987,9 @@ https://archiveweb.page/images/${"logo.svg"}`}
       : [];
     const additionalSeedUrlList = this.formState.urlList
       ? urlListToArray(this.formState.urlList).map((seedUrl) => {
-        const newSeed: Seed = {url: seedUrl, scopeType: "page"};
+          const newSeed: Seed = { url: seedUrl, scopeType: "page" };
-        return newSeed;
+          return newSeed;
-      })
+        })
       : [];
     const primarySeed: Seed = {
       url: primarySeedUrl,
@@ -2016,18 +2043,26 @@ https://archiveweb.page/images/${"logo.svg"}`}
   }
 
   private async fetchAPIDefaults() {
+    const orgDefaults = { ...this.orgDefaults };
     try {
-      const data = await this.apiFetch("/settings", this.authState!);
+      const resp = await fetch("/api/settings", {
+        headers: { "Content-Type": "application/json" },
+      });
+      if (!resp.ok) {
+        throw new Error(resp.statusText);
+      }
+      const data = await resp.json();
       if (data.defaultBehaviorTimeSeconds) {
-        this.defaultBehaviorTimeoutMinutes =
+        orgDefaults.behaviorTimeoutMinutes =
           data.defaultBehaviorTimeSeconds / 60;
-      } else {
+      }
-        this.defaultBehaviorTimeoutMinutes = DEFAULT_BEHAVIOR_TIMEOUT_MINUTES;
+      if (data.maxPagesPerCrawl > 0) {
+        orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl;
       }
     } catch (e: any) {
       console.debug(e);
-      this.defaultBehaviorTimeoutMinutes = DEFAULT_BEHAVIOR_TIMEOUT_MINUTES;
     }
+    this.orgDefaults = orgDefaults;
   }
 }