backend api: add curr crawl to crawlconfig listing (#95)
* backend api: add current crawl id to crawlconfig listing - model: add 'currCrawlId' to CrawlConfig model - output: add response model to /crawlconfigs api response to show correct openapi model - rename crawl_configs -> crawlConfigs for consistency
This commit is contained in:
		
							parent
							
								
									ec1a758e42
								
							
						
					
					
						commit
						b506442b21
					
				| @ -110,6 +110,19 @@ class CrawlConfig(BaseMongoModel): | |||||||
|     lastCrawlTime: Optional[datetime] |     lastCrawlTime: Optional[datetime] | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | # ============================================================================ | ||||||
|  | class CrawlConfigOut(CrawlConfig): | ||||||
|  |     """Crawl Config Output, includes currCrawlId of running crawl""" | ||||||
|  | 
 | ||||||
|  |     currCrawlId: Optional[str] | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | # ============================================================================ | ||||||
|  | class CrawlConfigsResponse(BaseModel): | ||||||
|  |     """ model for crawl configs response """ | ||||||
|  |     crawlConfigs: List[CrawlConfigOut] | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
| # ============================================================================ | # ============================================================================ | ||||||
| class UpdateSchedule(BaseModel): | class UpdateSchedule(BaseModel): | ||||||
|     """ Update the crawl schedule """ |     """ Update the crawl schedule """ | ||||||
| @ -187,7 +200,21 @@ class CrawlOps: | |||||||
|         """Get all crawl configs for an archive is a member of""" |         """Get all crawl configs for an archive is a member of""" | ||||||
|         cursor = self.crawl_configs.find({"archive": archive.id}) |         cursor = self.crawl_configs.find({"archive": archive.id}) | ||||||
|         results = await cursor.to_list(length=1000) |         results = await cursor.to_list(length=1000) | ||||||
|         return [CrawlConfig.from_dict(res) for res in results] | 
 | ||||||
|  |         crawls = await self.crawl_manager.list_running_crawls(aid=archive.id) | ||||||
|  | 
 | ||||||
|  |         running = {} | ||||||
|  |         for crawl in crawls: | ||||||
|  |             running[crawl.cid] = crawl.id | ||||||
|  | 
 | ||||||
|  |         configs = [] | ||||||
|  |         for res in results: | ||||||
|  |             config = CrawlConfigOut.from_dict(res) | ||||||
|  |             # pylint: disable=invalid-name | ||||||
|  |             config.currCrawlId = running.get(config.id) | ||||||
|  |             configs.append(config) | ||||||
|  | 
 | ||||||
|  |         return configs | ||||||
| 
 | 
 | ||||||
|     async def get_crawl_config(self, cid: str, archive: Archive): |     async def get_crawl_config(self, cid: str, archive: Archive): | ||||||
|         """Get an archive for user by unique id""" |         """Get an archive for user by unique id""" | ||||||
| @ -226,12 +253,13 @@ def init_crawl_config_api(mdb, user_dep, archive_ops, crawl_manager): | |||||||
| 
 | 
 | ||||||
|         return archive |         return archive | ||||||
| 
 | 
 | ||||||
|     @router.get("") |     @router.get("", response_model=CrawlConfigsResponse) | ||||||
|     async def get_crawl_configs(archive: Archive = Depends(archive_crawl_dep)): |     async def get_crawl_configs(archive: Archive = Depends(archive_crawl_dep)): | ||||||
|         results = await ops.get_crawl_configs(archive) |         results = await ops.get_crawl_configs(archive) | ||||||
|         return { |         return CrawlConfigsResponse(crawlConfigs=results) | ||||||
|             "crawl_configs": [res.serialize(exclude={"archive"}) for res in results] |         # return { | ||||||
|         } |         #    "crawl_configs": [res.serialize(exclude={"archive"}) for res in results] | ||||||
|  |         # } | ||||||
| 
 | 
 | ||||||
|     @router.get("/{cid}") |     @router.get("/{cid}") | ||||||
|     async def get_crawl_config(crawl_config: CrawlConfig = Depends(crawls_dep)): |     async def get_crawl_config(crawl_config: CrawlConfig = Depends(crawls_dep)): | ||||||
|  | |||||||
		Loading…
	
		Reference in New Issue
	
	Block a user