`;
@@ -1383,7 +1523,7 @@ https://example.net`}
return html`
${errorAlert}
-
+
${when(this.progressState.activeTab === "confirmSettings", () => {
// Prevent parsing and rendering tab when not visible
const crawlConfig = this.parseConfig();
@@ -1515,14 +1655,22 @@ https://example.net`}
};
private syncTabErrorState(el: HTMLElement) {
- const currentTab = this.progressState.activeTab as StepName;
const panelEl = el.closest("btrix-tab-panel")!;
+ const tabName = panelEl
+ .getAttribute("name")!
+ .replace("newJobConfig-", "") as StepName;
const hasInvalid = panelEl.querySelector("[data-user-invalid]");
- if (!hasInvalid && this.progressState.tabs[currentTab].error) {
+ if (!hasInvalid && this.progressState.tabs[tabName].error) {
this.updateProgressState({
tabs: {
- [currentTab]: { error: false },
+ [tabName]: { error: false },
+ },
+ });
+ } else if (hasInvalid && !this.progressState.tabs[tabName].error) {
+ this.updateProgressState({
+ tabs: {
+ [tabName]: { error: true },
},
});
}
@@ -1541,7 +1689,7 @@ https://example.net`}
value = elem.value;
break;
case "sl-input": {
- if ((elem as SlInput).type === "number") {
+ if ((elem as SlInput).type === "number" && elem.value !== "") {
value = +elem.value;
} else {
value = elem.value;
@@ -1613,16 +1761,19 @@ https://example.net`}
const el = event.target as HTMLElement;
const tagName = el.tagName.toLowerCase();
if (tagName !== "sl-input") return;
-
const { key } = event;
if ((el as SlInput).type === "number") {
// Prevent typing non-numeric keys
- if (key.length === 1 && /\D/.test(key)) {
+ if (
+ !event.metaKey &&
+ !event.shiftKey &&
+ key.length === 1 &&
+ /\D/.test(key)
+ ) {
event.preventDefault();
return;
}
}
-
if (
key === "Enter" &&
this.progressState.activeTab !== STEPS[STEPS.length - 1]
@@ -1642,7 +1793,6 @@ https://example.net`}
}
const config = this.parseConfig();
-
this.isSubmitting = true;
try {
@@ -1768,10 +1918,9 @@ https://example.net`}
this.defaultBehaviorTimeoutMinutes ??
DEFAULT_BEHAVIOR_TIMEOUT_MINUTES) * 60,
limit: this.formState.pageLimit ? +this.formState.pageLimit : null,
- extraHops: this.formState.includeLinkedPages ? 1 : 0,
lang: this.formState.lang || null,
blockAds: this.formState.blockAds,
- exclude: trimExclusions(this.formState.exclusions),
+ exclude: trimArray(this.formState.exclusions),
},
};
@@ -1786,6 +1935,7 @@ https://example.net`}
const config = {
seeds: urlListToArray(this.formState.urlList),
scopeType: "page" as FormState["scopeType"],
+ extraHops: this.formState.includeLinkedPages ? 1 : 0,
};
return config;
@@ -1793,43 +1943,33 @@ https://example.net`}
private parseSeededConfig(): NewCrawlConfigParams["config"] {
const primarySeedUrl = this.formState.primarySeedUrl.replace(/\/$/, "");
- const externalUrlList = this.formState.allowedExternalUrlList
- ? urlListToArray(this.formState.allowedExternalUrlList).map((str) =>
+ const includeUrlList = this.formState.customIncludeUrlList
+ ? urlListToArray(this.formState.customIncludeUrlList).map((str) =>
str.replace(/\/$/, "")
)
: [];
- let scopeType = this.formState.scopeType;
- const include = [];
- if (externalUrlList.length) {
- const { host, origin } = new URL(primarySeedUrl);
- scopeType = "custom";
-
- // Replicate scope type with regex
- switch (this.formState.scopeType) {
- case "prefix":
- include.push(`${regexEscape(primarySeedUrl)}\/.*`);
- break;
- case "host":
- include.push(`${regexEscape(origin)}\/.*`);
- break;
- case "domain":
- include.push(
- `${regexEscape(origin)}\/.*`,
- `.*\.${regexEscape(host)}\/.*`
- );
- break;
- default:
- break;
- }
-
- externalUrlList.forEach((url) => {
- include.push(`${regexEscape(url)}\/.*`);
- });
- }
- const config = {
- seeds: [primarySeedUrl],
- scopeType,
- include,
+ const additionalSeedUrlList = this.formState.urlList
+ ? urlListToArray(this.formState.urlList).map((str) =>
+ str.replace(/\/$/, "")
+ )
+ : [];
+ const primarySeed: Seed = {
+ url: primarySeedUrl,
+ scopeType: this.formState.scopeType,
+ include:
+ this.formState.scopeType === "custom"
+ ? [
+ `${regexEscape(primarySeedUrl)}\/.*`,
+ ...includeUrlList.map((url) => `${regexEscape(url)}\/.*`),
+ ]
+ : [],
+ extraHops: this.formState.includeLinkedPages ? 1 : 0,
+ };
+ const config: SeedConfig = {
+ seeds: [primarySeed, ...additionalSeedUrlList],
+ scopeType: additionalSeedUrlList.length
+ ? "page"
+ : this.formState.scopeType,
};
return config;
}
diff --git a/frontend/src/pages/org/types.ts b/frontend/src/pages/org/types.ts
index 49c4598e..ba23172a 100644
--- a/frontend/src/pages/org/types.ts
+++ b/frontend/src/pages/org/types.ts
@@ -29,23 +29,29 @@ export type Crawl = {
tags?: string[];
};
+type ScopeType =
+ | "prefix"
+ | "host"
+ | "domain"
+ | "page"
+ | "page-spa"
+ | "any"
+ | "custom";
+
export type Seed = {
- scopeType:
- | "prefix"
- | "host"
- | "domain"
- | "page"
- | "page-spa"
- | "any"
- | "custom";
+ url: string;
+ scopeType: ScopeType;
include?: string[];
exclude?: string[];
limit?: number | null;
+ extraHops?: number | null;
};
-export type SeedConfig = Seed & {
- seeds: (string | ({ url: string } & Seed))[];
- extraHops?: number | null;
+export type SeedConfig = Pick<
+ Seed,
+ "scopeType" | "include" | "exclude" | "limit" | "extraHops"
+> & {
+ seeds: (string | Seed)[];
lang?: string | null;
blockAds?: boolean;
behaviorTimeout?: number | null;
diff --git a/frontend/src/shoelace.ts b/frontend/src/shoelace.ts
index 20462435..3327c14a 100644
--- a/frontend/src/shoelace.ts
+++ b/frontend/src/shoelace.ts
@@ -18,6 +18,7 @@ import "@shoelace-style/shoelace/dist/components/resize-observer/resize-observer
import "@shoelace-style/shoelace/dist/components/select/select";
import "@shoelace-style/shoelace/dist/components/switch/switch";
import "@shoelace-style/shoelace/dist/components/textarea/textarea";
+import "@shoelace-style/shoelace/dist/components/mutation-observer/mutation-observer";
import(
/* webpackChunkName: "shoelace" */ "@shoelace-style/shoelace/dist/components/dialog/dialog"
);