diff --git a/backend/btrixcloud/k8sapi.py b/backend/btrixcloud/k8sapi.py index 46c1427f..740a90d7 100644 --- a/backend/btrixcloud/k8sapi.py +++ b/backend/btrixcloud/k8sapi.py @@ -98,8 +98,8 @@ class K8sAPI: "oid": oid, "userid": userid, "scale": scale, - "expire_time": crawl_expire_time, - "max_crawl_size": max_crawl_size, + "expire_time": crawl_expire_time or 0, + "max_crawl_size": max_crawl_size or 0, "manual": "1" if manual else "0", } diff --git a/backend/btrixcloud/models.py b/backend/btrixcloud/models.py index 6f651e14..d6bee42d 100644 --- a/backend/btrixcloud/models.py +++ b/backend/btrixcloud/models.py @@ -128,8 +128,8 @@ class CrawlConfigIn(BaseModel): autoAddCollections: Optional[List[UUID4]] = [] tags: Optional[List[str]] = [] - crawlTimeout: Optional[int] = 0 - maxCrawlSize: Optional[int] = 0 + crawlTimeout: int = 0 + maxCrawlSize: int = 0 scale: Optional[conint(ge=1, le=MAX_CRAWL_SCALE)] = 1 crawlFilenameTemplate: Optional[str] diff --git a/backend/btrixcloud/operator.py b/backend/btrixcloud/operator.py index 8dafd808..8a55435e 100644 --- a/backend/btrixcloud/operator.py +++ b/backend/btrixcloud/operator.py @@ -280,7 +280,7 @@ class BtrixOperator(K8sAPI): started=data.parent["metadata"]["creationTimestamp"], stopping=spec.get("stopping", False), expire_time=from_k8s_date(spec.get("expireTime")), - max_crawl_size=int(configmap.get("MAX_CRAWL_SIZE", "0")), + max_crawl_size=int(spec.get("maxCrawlSize") or 0), scheduled=spec.get("manual") != "1", ) diff --git a/frontend/src/pages/org/workflow-editor.ts b/frontend/src/pages/org/workflow-editor.ts index d5b99289..c5d906be 100644 --- a/frontend/src/pages/org/workflow-editor.ts +++ b/frontend/src/pages/org/workflow-editor.ts @@ -78,11 +78,11 @@ type FormState = { includeLinkedPages: boolean; useSitemap: boolean; customIncludeUrlList: string; - crawlTimeoutMinutes: number | null; + crawlTimeoutMinutes: number; behaviorTimeoutSeconds: number | null; pageLoadTimeoutSeconds: number | null; pageExtraDelaySeconds: number | null; - maxCrawlSizeGB: number | null; + maxCrawlSizeGB: number; maxScopeDepth: number | null; scopeType: WorkflowParams["config"]["scopeType"]; exclusions: WorkflowParams["config"]["exclude"]; @@ -153,7 +153,7 @@ const getDefaultFormState = (): FormState => ({ includeLinkedPages: false, useSitemap: true, customIncludeUrlList: "", - crawlTimeoutMinutes: null, + crawlTimeoutMinutes: 0, maxCrawlSizeGB: 0, behaviorTimeoutSeconds: null, pageLoadTimeoutSeconds: null, @@ -488,12 +488,12 @@ export class CrawlConfigEditor extends LiteElement { formState.autoAddCollections = this.initialWorkflow.autoAddCollections; } - const secondsToMinutes = (value: any, fallback: number | null) => { + const secondsToMinutes = (value: any, fallback: number = 0) => { if (typeof value === "number" && value > 0) return value / 60; return fallback; }; - const bytesToGB = (value: any, fallback: number | null) => { + const bytesToGB = (value: any, fallback: number = 0) => { if (typeof value === "number" && value > 0) return Math.floor(value / BYTES_PER_GB); return fallback; @@ -2147,12 +2147,8 @@ https://archiveweb.page/images/${"logo.svg"}`} profileid: this.formState.browserProfile?.id || "", runNow: this.formState.runNow || this.formState.scheduleType === "now", schedule: this.formState.scheduleType === "cron" ? this.utcSchedule : "", - crawlTimeout: this.formState.crawlTimeoutMinutes - ? this.formState.crawlTimeoutMinutes * 60 - : null, - maxCrawlSize: this.formState.maxCrawlSizeGB - ? this.formState.maxCrawlSizeGB * BYTES_PER_GB - : null, + crawlTimeout: this.formState.crawlTimeoutMinutes * 60, + maxCrawlSize: this.formState.maxCrawlSizeGB * BYTES_PER_GB, tags: this.formState.tags, autoAddCollections: this.formState.autoAddCollections, config: {