Set max pages to API default (#739)
This commit is contained in:
parent
2b0d5ff8b3
commit
370b8cbd4d
@ -29,6 +29,12 @@ export class ConfigDetails extends LiteElement {
|
||||
@property({ type: Boolean })
|
||||
hideTags = false;
|
||||
|
||||
@state()
|
||||
private orgDefaults = {
|
||||
behaviorTimeoutMinutes: Infinity,
|
||||
maxPagesPerCrawl: Infinity,
|
||||
};
|
||||
|
||||
private readonly scopeTypeLabels: Record<
|
||||
CrawlConfig["config"]["scopeType"],
|
||||
string
|
||||
@ -42,6 +48,11 @@ export class ConfigDetails extends LiteElement {
|
||||
any: msg("Any"),
|
||||
};
|
||||
|
||||
connectedCallback() {
|
||||
super.connectedCallback();
|
||||
this.fetchAPIDefaults();
|
||||
}
|
||||
|
||||
render() {
|
||||
const crawlConfig = this.crawlConfig;
|
||||
const exclusions = crawlConfig?.config.exclude || [];
|
||||
@ -204,6 +215,7 @@ export class ConfigDetails extends LiteElement {
|
||||
let primarySeedConfig: SeedConfig | Seed = seedsConfig;
|
||||
let primarySeedUrl = seedsConfig.seeds[0].url;
|
||||
const includeUrlList = primarySeedConfig.include || seedsConfig.include;
|
||||
const maxPages = primarySeedConfig.limit ?? seedsConfig.limit;
|
||||
return html`
|
||||
${this.renderSetting(msg("Primary Seed URL"), primarySeedUrl, true)}
|
||||
${this.renderSetting(
|
||||
@ -245,9 +257,16 @@ export class ConfigDetails extends LiteElement {
|
||||
)}
|
||||
${this.renderSetting(
|
||||
msg("Max Pages"),
|
||||
seedsConfig.limit
|
||||
? msg(str`${primarySeedConfig.limit ?? seedsConfig.limit} pages`)
|
||||
: msg("Unlimited")
|
||||
when(
|
||||
maxPages,
|
||||
() => msg(str`${maxPages} page(s)`),
|
||||
() =>
|
||||
this.orgDefaults.maxPagesPerCrawl < Infinity
|
||||
? msg(
|
||||
str`Maximum Allowed (${this.orgDefaults.maxPagesPerCrawl.toLocaleString()} pages)`
|
||||
)
|
||||
: undefined
|
||||
)
|
||||
)}
|
||||
`;
|
||||
};
|
||||
@ -283,4 +302,27 @@ export class ConfigDetails extends LiteElement {
|
||||
</btrix-desc-list-item>
|
||||
`;
|
||||
}
|
||||
|
||||
private async fetchAPIDefaults() {
|
||||
const orgDefaults = { ...this.orgDefaults };
|
||||
try {
|
||||
const resp = await fetch("/api/settings", {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
if (!resp.ok) {
|
||||
throw new Error(resp.statusText);
|
||||
}
|
||||
const data = await resp.json();
|
||||
if (data.defaultBehaviorTimeSeconds) {
|
||||
orgDefaults.behaviorTimeoutMinutes =
|
||||
data.defaultBehaviorTimeSeconds / 60;
|
||||
}
|
||||
if (data.maxPagesPerCrawl > 0) {
|
||||
orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl;
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.debug(e);
|
||||
}
|
||||
this.orgDefaults = orgDefaults;
|
||||
}
|
||||
}
|
||||
|
@ -192,6 +192,7 @@ const urlListToArray = flow(
|
||||
trimArray
|
||||
);
|
||||
const DEFAULT_BEHAVIOR_TIMEOUT_MINUTES = 5;
|
||||
const DEFAULT_MAX_PAGES_PER_CRAWL = Infinity;
|
||||
|
||||
@localized()
|
||||
export class CrawlConfigEditor extends LiteElement {
|
||||
@ -220,7 +221,10 @@ export class CrawlConfigEditor extends LiteElement {
|
||||
private progressState!: ProgressState;
|
||||
|
||||
@state()
|
||||
private defaultBehaviorTimeoutMinutes?: number;
|
||||
private orgDefaults = {
|
||||
behaviorTimeoutMinutes: DEFAULT_BEHAVIOR_TIMEOUT_MINUTES,
|
||||
maxPagesPerCrawl: DEFAULT_MAX_PAGES_PER_CRAWL,
|
||||
};
|
||||
|
||||
@state()
|
||||
private formState!: FormState;
|
||||
@ -1172,14 +1176,37 @@ https://archiveweb.page/images/${"logo.svg"}`}
|
||||
type="number"
|
||||
value=${this.formState.pageLimit || ""}
|
||||
min=${minPages}
|
||||
placeholder=${msg("Unlimited")}
|
||||
max=${this.orgDefaults.maxPagesPerCrawl}
|
||||
placeholder=${this.orgDefaults.maxPagesPerCrawl === Infinity
|
||||
? msg("Unlimited")
|
||||
: msg(
|
||||
str`Maximum Allowed (${this.orgDefaults.maxPagesPerCrawl.toLocaleString()})`
|
||||
)}
|
||||
@sl-input=${async (e: CustomEvent) => {
|
||||
const inputEl = e.target as SlInput;
|
||||
await inputEl.updateComplete;
|
||||
let helpText = "";
|
||||
if (inputEl.invalid) {
|
||||
const value = +inputEl.value;
|
||||
if (value < minPages) {
|
||||
helpText =
|
||||
minPages === 1
|
||||
? msg(
|
||||
str`Minimum ${minPages.toLocaleString()} page per crawl`
|
||||
)
|
||||
: msg(
|
||||
str`Minimum ${minPages.toLocaleString()} pages per crawl`
|
||||
);
|
||||
} else if (value > this.orgDefaults.maxPagesPerCrawl) {
|
||||
helpText = msg(
|
||||
str`Maximum ${this.orgDefaults.maxPagesPerCrawl.toLocaleString()} pages per crawl`
|
||||
);
|
||||
}
|
||||
}
|
||||
inputEl.helpText = helpText;
|
||||
}}
|
||||
>
|
||||
<span slot="suffix">${msg("pages")}</span>
|
||||
<div slot="help-text">
|
||||
${minPages === 1
|
||||
? msg(str`Minimum ${minPages} page`)
|
||||
: msg(str`Minimum ${minPages} pages`)}
|
||||
</div>
|
||||
</sl-input>
|
||||
</sl-mutation-observer>
|
||||
`)}
|
||||
@ -1195,9 +1222,9 @@ https://archiveweb.page/images/${"logo.svg"}`}
|
||||
placeholder=${msg("Unlimited")}
|
||||
value=${ifDefined(
|
||||
this.formState.pageTimeoutMinutes ??
|
||||
this.defaultBehaviorTimeoutMinutes
|
||||
this.orgDefaults.behaviorTimeoutMinutes
|
||||
)}
|
||||
?disabled=${this.defaultBehaviorTimeoutMinutes === undefined}
|
||||
?disabled=${this.orgDefaults.behaviorTimeoutMinutes === undefined}
|
||||
min="1"
|
||||
required
|
||||
>
|
||||
@ -1928,7 +1955,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
|
||||
: this.parseUrlListConfig()),
|
||||
behaviorTimeout:
|
||||
(this.formState.pageTimeoutMinutes ??
|
||||
this.defaultBehaviorTimeoutMinutes ??
|
||||
this.orgDefaults.behaviorTimeoutMinutes ??
|
||||
DEFAULT_BEHAVIOR_TIMEOUT_MINUTES) * 60,
|
||||
limit: this.formState.pageLimit ? +this.formState.pageLimit : undefined,
|
||||
lang: this.formState.lang || "",
|
||||
@ -1943,7 +1970,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
|
||||
private parseUrlListConfig(): NewCrawlConfigParams["config"] {
|
||||
const config = {
|
||||
seeds: urlListToArray(this.formState.urlList).map((seedUrl) => {
|
||||
const newSeed: Seed = {url: seedUrl, scopeType: "page"};
|
||||
const newSeed: Seed = { url: seedUrl, scopeType: "page" };
|
||||
return newSeed;
|
||||
}),
|
||||
scopeType: "page" as FormState["scopeType"],
|
||||
@ -1960,9 +1987,9 @@ https://archiveweb.page/images/${"logo.svg"}`}
|
||||
: [];
|
||||
const additionalSeedUrlList = this.formState.urlList
|
||||
? urlListToArray(this.formState.urlList).map((seedUrl) => {
|
||||
const newSeed: Seed = {url: seedUrl, scopeType: "page"};
|
||||
return newSeed;
|
||||
})
|
||||
const newSeed: Seed = { url: seedUrl, scopeType: "page" };
|
||||
return newSeed;
|
||||
})
|
||||
: [];
|
||||
const primarySeed: Seed = {
|
||||
url: primarySeedUrl,
|
||||
@ -2016,18 +2043,26 @@ https://archiveweb.page/images/${"logo.svg"}`}
|
||||
}
|
||||
|
||||
private async fetchAPIDefaults() {
|
||||
const orgDefaults = { ...this.orgDefaults };
|
||||
try {
|
||||
const data = await this.apiFetch("/settings", this.authState!);
|
||||
const resp = await fetch("/api/settings", {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
if (!resp.ok) {
|
||||
throw new Error(resp.statusText);
|
||||
}
|
||||
const data = await resp.json();
|
||||
if (data.defaultBehaviorTimeSeconds) {
|
||||
this.defaultBehaviorTimeoutMinutes =
|
||||
orgDefaults.behaviorTimeoutMinutes =
|
||||
data.defaultBehaviorTimeSeconds / 60;
|
||||
} else {
|
||||
this.defaultBehaviorTimeoutMinutes = DEFAULT_BEHAVIOR_TIMEOUT_MINUTES;
|
||||
}
|
||||
if (data.maxPagesPerCrawl > 0) {
|
||||
orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl;
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.debug(e);
|
||||
this.defaultBehaviorTimeoutMinutes = DEFAULT_BEHAVIOR_TIMEOUT_MINUTES;
|
||||
}
|
||||
this.orgDefaults = orgDefaults;
|
||||
}
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user