From 36e723cc515a794545814b124e9ae5f9ff810c10 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Thu, 20 Feb 2025 11:03:28 -0800 Subject: [PATCH] Adjust crawler pvc on exit code 3 (out of storage) (#2375) crawler 1.5.0 now has an exit code 3 for when crawler is actually out of disk space. The operator should handle this by immediately adjusting the PVC size. Ideally, crawler will be improved to avoid this, but since this can still happen, operator should be able to respond and fix the issue. --- backend/btrixcloud/operator/crawls.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/backend/btrixcloud/operator/crawls.py b/backend/btrixcloud/operator/crawls.py index 172af560..a473a376 100644 --- a/backend/btrixcloud/operator/crawls.py +++ b/backend/btrixcloud/operator/crawls.py @@ -1381,6 +1381,7 @@ class CrawlOperator(BaseOperator): ) for key, value in sizes.items(): + increase_storage = False value = int(value) if value > 0 and status.podStatus: pod_info = status.podStatus[key] @@ -1393,15 +1394,21 @@ class CrawlOperator(BaseOperator): and pod_info.used.storage * self.min_avail_storage_ratio > pod_info.allocated.storage ): - new_storage = math.ceil( - pod_info.used.storage - * self.min_avail_storage_ratio - / 1_000_000_000 - ) - pod_info.newStorage = f"{new_storage}Gi" - print( - f"Attempting to adjust storage to {pod_info.newStorage} for {key}" - ) + increase_storage = True + + # out of storage + if pod_info.isNewExit and pod_info.exitCode == 3: + pod_info.used.storage = pod_info.allocated.storage + increase_storage = True + + if increase_storage: + new_storage = math.ceil( + pod_info.used.storage * self.min_avail_storage_ratio / 1_000_000_000 + ) + pod_info.newStorage = f"{new_storage}Gi" + print( + f"Attempting to adjust storage to {pod_info.newStorage} for {key}" + ) if not status.stopReason: status.stopReason = await self.is_crawl_stopping(crawl, status, data)