Adjust crawler pvc on exit code 3 (out of storage) (#2375)
crawler 1.5.0 now has an exit code 3 for when crawler is actually out of disk space. The operator should handle this by immediately adjusting the PVC size. Ideally, crawler will be improved to avoid this, but since this can still happen, operator should be able to respond and fix the issue.
This commit is contained in:
		
							parent
							
								
									88a9f3baf7
								
							
						
					
					
						commit
						36e723cc51
					
				| @ -1381,6 +1381,7 @@ class CrawlOperator(BaseOperator): | ||||
|         ) | ||||
| 
 | ||||
|         for key, value in sizes.items(): | ||||
|             increase_storage = False | ||||
|             value = int(value) | ||||
|             if value > 0 and status.podStatus: | ||||
|                 pod_info = status.podStatus[key] | ||||
| @ -1393,15 +1394,21 @@ class CrawlOperator(BaseOperator): | ||||
|                     and pod_info.used.storage * self.min_avail_storage_ratio | ||||
|                     > pod_info.allocated.storage | ||||
|                 ): | ||||
|                     new_storage = math.ceil( | ||||
|                         pod_info.used.storage | ||||
|                         * self.min_avail_storage_ratio | ||||
|                         / 1_000_000_000 | ||||
|                     ) | ||||
|                     pod_info.newStorage = f"{new_storage}Gi" | ||||
|                     print( | ||||
|                         f"Attempting to adjust storage to {pod_info.newStorage} for {key}" | ||||
|                     ) | ||||
|                     increase_storage = True | ||||
| 
 | ||||
|             # out of storage | ||||
|             if pod_info.isNewExit and pod_info.exitCode == 3: | ||||
|                 pod_info.used.storage = pod_info.allocated.storage | ||||
|                 increase_storage = True | ||||
| 
 | ||||
|             if increase_storage: | ||||
|                 new_storage = math.ceil( | ||||
|                     pod_info.used.storage * self.min_avail_storage_ratio / 1_000_000_000 | ||||
|                 ) | ||||
|                 pod_info.newStorage = f"{new_storage}Gi" | ||||
|                 print( | ||||
|                     f"Attempting to adjust storage to {pod_info.newStorage} for {key}" | ||||
|                 ) | ||||
| 
 | ||||
|         if not status.stopReason: | ||||
|             status.stopReason = await self.is_crawl_stopping(crawl, status, data) | ||||
|  | ||||
		Loading…
	
		Reference in New Issue
	
	Block a user