tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adjust crawler pvc on exit code 3 (out of storage) (#2375)

Browse Source 
crawler 1.5.0 now has an exit code 3 for when crawler is actually out of
disk space. The operator should handle this by immediately adjusting the
PVC size.

Ideally, crawler will be improved to avoid this, but since this can
still happen, operator should be able to respond and fix the issue.
This commit is contained in:
Ilya Kreymer 2025-02-20 11:03:28 -08:00 committed by GitHub
parent 88a9f3baf7
commit 36e723cc51
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
backend/btrixcloud/operator/crawls.py
View File

@ -1381,6 +1381,7 @@ class CrawlOperator(BaseOperator):
) )
for key, value in sizes.items(): for key, value in sizes.items():
increase_storage = False
value = int(value) value = int(value)
if value > 0 and status.podStatus: if value > 0 and status.podStatus:
pod_info = status.podStatus[key] pod_info = status.podStatus[key]
@ -1393,15 +1394,21 @@ class CrawlOperator(BaseOperator):
and pod_info.used.storage * self.min_avail_storage_ratio and pod_info.used.storage * self.min_avail_storage_ratio
> pod_info.allocated.storage > pod_info.allocated.storage
): ):
new_storage = math.ceil( increase_storage = True
pod_info.used.storage
* self.min_avail_storage_ratio # out of storage
/ 1_000_000_000 if pod_info.isNewExit and pod_info.exitCode == 3:
) pod_info.used.storage = pod_info.allocated.storage
pod_info.newStorage = f"{new_storage}Gi" increase_storage = True
print(
f"Attempting to adjust storage to {pod_info.newStorage} for {key}" if increase_storage:
) new_storage = math.ceil(
pod_info.used.storage * self.min_avail_storage_ratio / 1_000_000_000
)
pod_info.newStorage = f"{new_storage}Gi"
print(
f"Attempting to adjust storage to {pod_info.newStorage} for {key}"
)
if not status.stopReason: if not status.stopReason:
status.stopReason = await self.is_crawl_stopping(crawl, status, data) status.stopReason = await self.is_crawl_stopping(crawl, status, data)