Set max pages to API default (#739)

This commit is contained in:
sua yoo 2023-04-04 08:47:37 -07:00 committed by GitHub
parent 2b0d5ff8b3
commit 370b8cbd4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 99 additions and 22 deletions

View File

@ -29,6 +29,12 @@ export class ConfigDetails extends LiteElement {
@property({ type: Boolean })
hideTags = false;
@state()
private orgDefaults = {
behaviorTimeoutMinutes: Infinity,
maxPagesPerCrawl: Infinity,
};
private readonly scopeTypeLabels: Record<
CrawlConfig["config"]["scopeType"],
string
@ -42,6 +48,11 @@ export class ConfigDetails extends LiteElement {
any: msg("Any"),
};
connectedCallback() {
super.connectedCallback();
this.fetchAPIDefaults();
}
render() {
const crawlConfig = this.crawlConfig;
const exclusions = crawlConfig?.config.exclude || [];
@ -204,6 +215,7 @@ export class ConfigDetails extends LiteElement {
let primarySeedConfig: SeedConfig | Seed = seedsConfig;
let primarySeedUrl = seedsConfig.seeds[0].url;
const includeUrlList = primarySeedConfig.include || seedsConfig.include;
const maxPages = primarySeedConfig.limit ?? seedsConfig.limit;
return html`
${this.renderSetting(msg("Primary Seed URL"), primarySeedUrl, true)}
${this.renderSetting(
@ -245,9 +257,16 @@ export class ConfigDetails extends LiteElement {
)}
${this.renderSetting(
msg("Max Pages"),
seedsConfig.limit
? msg(str`${primarySeedConfig.limit ?? seedsConfig.limit} pages`)
: msg("Unlimited")
when(
maxPages,
() => msg(str`${maxPages} page(s)`),
() =>
this.orgDefaults.maxPagesPerCrawl < Infinity
? msg(
str`Maximum Allowed (${this.orgDefaults.maxPagesPerCrawl.toLocaleString()} pages)`
)
: undefined
)
)}
`;
};
@ -283,4 +302,27 @@ export class ConfigDetails extends LiteElement {
</btrix-desc-list-item>
`;
}
private async fetchAPIDefaults() {
const orgDefaults = { ...this.orgDefaults };
try {
const resp = await fetch("/api/settings", {
headers: { "Content-Type": "application/json" },
});
if (!resp.ok) {
throw new Error(resp.statusText);
}
const data = await resp.json();
if (data.defaultBehaviorTimeSeconds) {
orgDefaults.behaviorTimeoutMinutes =
data.defaultBehaviorTimeSeconds / 60;
}
if (data.maxPagesPerCrawl > 0) {
orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl;
}
} catch (e: any) {
console.debug(e);
}
this.orgDefaults = orgDefaults;
}
}

View File

@ -192,6 +192,7 @@ const urlListToArray = flow(
trimArray
);
const DEFAULT_BEHAVIOR_TIMEOUT_MINUTES = 5;
const DEFAULT_MAX_PAGES_PER_CRAWL = Infinity;
@localized()
export class CrawlConfigEditor extends LiteElement {
@ -220,7 +221,10 @@ export class CrawlConfigEditor extends LiteElement {
private progressState!: ProgressState;
@state()
private defaultBehaviorTimeoutMinutes?: number;
private orgDefaults = {
behaviorTimeoutMinutes: DEFAULT_BEHAVIOR_TIMEOUT_MINUTES,
maxPagesPerCrawl: DEFAULT_MAX_PAGES_PER_CRAWL,
};
@state()
private formState!: FormState;
@ -1172,14 +1176,37 @@ https://archiveweb.page/images/${"logo.svg"}`}
type="number"
value=${this.formState.pageLimit || ""}
min=${minPages}
placeholder=${msg("Unlimited")}
max=${this.orgDefaults.maxPagesPerCrawl}
placeholder=${this.orgDefaults.maxPagesPerCrawl === Infinity
? msg("Unlimited")
: msg(
str`Maximum Allowed (${this.orgDefaults.maxPagesPerCrawl.toLocaleString()})`
)}
@sl-input=${async (e: CustomEvent) => {
const inputEl = e.target as SlInput;
await inputEl.updateComplete;
let helpText = "";
if (inputEl.invalid) {
const value = +inputEl.value;
if (value < minPages) {
helpText =
minPages === 1
? msg(
str`Minimum ${minPages.toLocaleString()} page per crawl`
)
: msg(
str`Minimum ${minPages.toLocaleString()} pages per crawl`
);
} else if (value > this.orgDefaults.maxPagesPerCrawl) {
helpText = msg(
str`Maximum ${this.orgDefaults.maxPagesPerCrawl.toLocaleString()} pages per crawl`
);
}
}
inputEl.helpText = helpText;
}}
>
<span slot="suffix">${msg("pages")}</span>
<div slot="help-text">
${minPages === 1
? msg(str`Minimum ${minPages} page`)
: msg(str`Minimum ${minPages} pages`)}
</div>
</sl-input>
</sl-mutation-observer>
`)}
@ -1195,9 +1222,9 @@ https://archiveweb.page/images/${"logo.svg"}`}
placeholder=${msg("Unlimited")}
value=${ifDefined(
this.formState.pageTimeoutMinutes ??
this.defaultBehaviorTimeoutMinutes
this.orgDefaults.behaviorTimeoutMinutes
)}
?disabled=${this.defaultBehaviorTimeoutMinutes === undefined}
?disabled=${this.orgDefaults.behaviorTimeoutMinutes === undefined}
min="1"
required
>
@ -1928,7 +1955,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
: this.parseUrlListConfig()),
behaviorTimeout:
(this.formState.pageTimeoutMinutes ??
this.defaultBehaviorTimeoutMinutes ??
this.orgDefaults.behaviorTimeoutMinutes ??
DEFAULT_BEHAVIOR_TIMEOUT_MINUTES) * 60,
limit: this.formState.pageLimit ? +this.formState.pageLimit : undefined,
lang: this.formState.lang || "",
@ -1943,7 +1970,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
private parseUrlListConfig(): NewCrawlConfigParams["config"] {
const config = {
seeds: urlListToArray(this.formState.urlList).map((seedUrl) => {
const newSeed: Seed = {url: seedUrl, scopeType: "page"};
const newSeed: Seed = { url: seedUrl, scopeType: "page" };
return newSeed;
}),
scopeType: "page" as FormState["scopeType"],
@ -1960,9 +1987,9 @@ https://archiveweb.page/images/${"logo.svg"}`}
: [];
const additionalSeedUrlList = this.formState.urlList
? urlListToArray(this.formState.urlList).map((seedUrl) => {
const newSeed: Seed = {url: seedUrl, scopeType: "page"};
return newSeed;
})
const newSeed: Seed = { url: seedUrl, scopeType: "page" };
return newSeed;
})
: [];
const primarySeed: Seed = {
url: primarySeedUrl,
@ -2016,18 +2043,26 @@ https://archiveweb.page/images/${"logo.svg"}`}
}
private async fetchAPIDefaults() {
const orgDefaults = { ...this.orgDefaults };
try {
const data = await this.apiFetch("/settings", this.authState!);
const resp = await fetch("/api/settings", {
headers: { "Content-Type": "application/json" },
});
if (!resp.ok) {
throw new Error(resp.statusText);
}
const data = await resp.json();
if (data.defaultBehaviorTimeSeconds) {
this.defaultBehaviorTimeoutMinutes =
orgDefaults.behaviorTimeoutMinutes =
data.defaultBehaviorTimeSeconds / 60;
} else {
this.defaultBehaviorTimeoutMinutes = DEFAULT_BEHAVIOR_TIMEOUT_MINUTES;
}
if (data.maxPagesPerCrawl > 0) {
orgDefaults.maxPagesPerCrawl = data.maxPagesPerCrawl;
}
} catch (e: any) {
console.debug(e);
this.defaultBehaviorTimeoutMinutes = DEFAULT_BEHAVIOR_TIMEOUT_MINUTES;
}
this.orgDefaults = orgDefaults;
}
}