backend api: add curr crawl to crawlconfig listing (#95)

* backend api: add current crawl id to crawlconfig listing
- model: add 'currCrawlId' to CrawlConfig model
- output: add response model to /crawlconfigs api response to show correct openapi model
- rename crawl_configs -> crawlConfigs for consistency
This commit is contained in:
Ilya Kreymer 2022-01-22 13:52:46 -08:00 committed by GitHub
parent ec1a758e42
commit b506442b21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -110,6 +110,19 @@ class CrawlConfig(BaseMongoModel):
lastCrawlTime: Optional[datetime]
# ============================================================================
class CrawlConfigOut(CrawlConfig):
"""Crawl Config Output, includes currCrawlId of running crawl"""
currCrawlId: Optional[str]
# ============================================================================
class CrawlConfigsResponse(BaseModel):
""" model for crawl configs response """
crawlConfigs: List[CrawlConfigOut]
# ============================================================================
class UpdateSchedule(BaseModel):
""" Update the crawl schedule """
@ -187,7 +200,21 @@ class CrawlOps:
"""Get all crawl configs for an archive is a member of"""
cursor = self.crawl_configs.find({"archive": archive.id})
results = await cursor.to_list(length=1000)
return [CrawlConfig.from_dict(res) for res in results]
crawls = await self.crawl_manager.list_running_crawls(aid=archive.id)
running = {}
for crawl in crawls:
running[crawl.cid] = crawl.id
configs = []
for res in results:
config = CrawlConfigOut.from_dict(res)
# pylint: disable=invalid-name
config.currCrawlId = running.get(config.id)
configs.append(config)
return configs
async def get_crawl_config(self, cid: str, archive: Archive):
"""Get an archive for user by unique id"""
@ -226,12 +253,13 @@ def init_crawl_config_api(mdb, user_dep, archive_ops, crawl_manager):
return archive
@router.get("")
@router.get("", response_model=CrawlConfigsResponse)
async def get_crawl_configs(archive: Archive = Depends(archive_crawl_dep)):
results = await ops.get_crawl_configs(archive)
return {
"crawl_configs": [res.serialize(exclude={"archive"}) for res in results]
}
return CrawlConfigsResponse(crawlConfigs=results)
# return {
# "crawl_configs": [res.serialize(exclude={"archive"}) for res in results]
# }
@router.get("/{cid}")
async def get_crawl_config(crawl_config: CrawlConfig = Depends(crawls_dep)):