tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix additional URLs (#752)

Browse Source
This commit is contained in:
sua yoo 2023-04-05 20:11:09 -07:00 committed by GitHub
parent 91c2c1ad62
commit 80bc4a3eb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
frontend/src/components/config-details.ts
View File

@ -267,9 +267,10 @@ export class ConfigDetails extends LiteElement {
const crawlConfig = this.crawlConfig!; const crawlConfig = this.crawlConfig!;
const seedsConfig = crawlConfig.config; const seedsConfig = crawlConfig.config;
const additionalUrlList = seedsConfig.seeds.slice(1); const additionalUrlList = seedsConfig.seeds.slice(1);
const primarySeedConfig: SeedConfig | Seed = seedsConfig; const primarySeedConfig: SeedConfig | Seed = seedsConfig.seeds[0];
const primarySeedUrl = seedsConfig.seeds[0].url; const primarySeedUrl = primarySeedConfig.url;
const includeUrlList = primarySeedConfig.include || seedsConfig.include; const includeUrlList =
primarySeedConfig.include || seedsConfig.include || [];
return html` return html`
${this.renderSetting(msg("Primary Seed URL"), primarySeedUrl, true)} ${this.renderSetting(msg("Primary Seed URL"), primarySeedUrl, true)}
${this.renderSetting( ${this.renderSetting(
@ -303,7 +304,10 @@ export class ConfigDetails extends LiteElement {
additionalUrlList?.length additionalUrlList?.length
? html` ? html`
<ul> <ul>
${additionalUrlList.map((url) => html`<li>${url}</li>`)} ${additionalUrlList.map(
(seed) =>
html`<li>${typeof seed === "string" ? seed : seed.url}</li>`
)}
</ul> </ul>
` `
: msg("None"), : msg("None"),

8
frontend/src/pages/org/workflow-editor.ts
View File

@ -195,6 +195,8 @@ const urlListToArray = flow(
(str: string) => (str.length ? str.trim().split(/\s+/g) : []), (str: string) => (str.length ? str.trim().split(/\s+/g) : []),
trimArray trimArray
); );
const mapSeedToUrl = (arr: Seed[]) =>
arr.map((seed) => (typeof seed === "string" ? seed : seed.url));
const DEFAULT_BEHAVIORS = [ const DEFAULT_BEHAVIORS = [
"autoscroll", "autoscroll",
"autoplay", "autoplay",
@ -428,13 +430,11 @@ export class CrawlConfigEditor extends LiteElement {
} }
const additionalSeeds = seeds.slice(1); const additionalSeeds = seeds.slice(1);
if (additionalSeeds.length) { if (additionalSeeds.length) {
formState.urlList = additionalSeeds.join("\n"); formState.urlList = mapSeedToUrl(additionalSeeds).join("\n");
} }
} else { } else {
// Treat "custom" like URL list // Treat "custom" like URL list
formState.urlList = seeds formState.urlList = mapSeedToUrl(seeds).join("\n");
.map((seed) => (typeof seed === "string" ? seed : seed.url))
.join("\n");
if (this.initialWorkflow.jobType === "custom") { if (this.initialWorkflow.jobType === "custom") {
formState.scopeType = seedsConfig.scopeType || "page"; formState.scopeType = seedsConfig.scopeType || "page";

4
frontend/src/types/crawler.ts
View File

@ -10,8 +10,8 @@ type ScopeType =
export type Seed = { export type Seed = {
url: string; url: string;
scopeType: ScopeType; scopeType: ScopeType;
include?: string[]; include?: string[] | null;
exclude?: string[]; exclude?: string[] | null;
limit?: number | null; limit?: number | null;
extraHops?: number | null; extraHops?: number | null;
}; };