backend api: add curr crawl to crawlconfig listing (#95)
* backend api: add current crawl id to crawlconfig listing - model: add 'currCrawlId' to CrawlConfig model - output: add response model to /crawlconfigs api response to show correct openapi model - rename crawl_configs -> crawlConfigs for consistency
This commit is contained in:
parent
ec1a758e42
commit
b506442b21
@ -110,6 +110,19 @@ class CrawlConfig(BaseMongoModel):
|
|||||||
lastCrawlTime: Optional[datetime]
|
lastCrawlTime: Optional[datetime]
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
class CrawlConfigOut(CrawlConfig):
|
||||||
|
"""Crawl Config Output, includes currCrawlId of running crawl"""
|
||||||
|
|
||||||
|
currCrawlId: Optional[str]
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
class CrawlConfigsResponse(BaseModel):
|
||||||
|
""" model for crawl configs response """
|
||||||
|
crawlConfigs: List[CrawlConfigOut]
|
||||||
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
class UpdateSchedule(BaseModel):
|
class UpdateSchedule(BaseModel):
|
||||||
""" Update the crawl schedule """
|
""" Update the crawl schedule """
|
||||||
@ -187,7 +200,21 @@ class CrawlOps:
|
|||||||
"""Get all crawl configs for an archive is a member of"""
|
"""Get all crawl configs for an archive is a member of"""
|
||||||
cursor = self.crawl_configs.find({"archive": archive.id})
|
cursor = self.crawl_configs.find({"archive": archive.id})
|
||||||
results = await cursor.to_list(length=1000)
|
results = await cursor.to_list(length=1000)
|
||||||
return [CrawlConfig.from_dict(res) for res in results]
|
|
||||||
|
crawls = await self.crawl_manager.list_running_crawls(aid=archive.id)
|
||||||
|
|
||||||
|
running = {}
|
||||||
|
for crawl in crawls:
|
||||||
|
running[crawl.cid] = crawl.id
|
||||||
|
|
||||||
|
configs = []
|
||||||
|
for res in results:
|
||||||
|
config = CrawlConfigOut.from_dict(res)
|
||||||
|
# pylint: disable=invalid-name
|
||||||
|
config.currCrawlId = running.get(config.id)
|
||||||
|
configs.append(config)
|
||||||
|
|
||||||
|
return configs
|
||||||
|
|
||||||
async def get_crawl_config(self, cid: str, archive: Archive):
|
async def get_crawl_config(self, cid: str, archive: Archive):
|
||||||
"""Get an archive for user by unique id"""
|
"""Get an archive for user by unique id"""
|
||||||
@ -226,12 +253,13 @@ def init_crawl_config_api(mdb, user_dep, archive_ops, crawl_manager):
|
|||||||
|
|
||||||
return archive
|
return archive
|
||||||
|
|
||||||
@router.get("")
|
@router.get("", response_model=CrawlConfigsResponse)
|
||||||
async def get_crawl_configs(archive: Archive = Depends(archive_crawl_dep)):
|
async def get_crawl_configs(archive: Archive = Depends(archive_crawl_dep)):
|
||||||
results = await ops.get_crawl_configs(archive)
|
results = await ops.get_crawl_configs(archive)
|
||||||
return {
|
return CrawlConfigsResponse(crawlConfigs=results)
|
||||||
"crawl_configs": [res.serialize(exclude={"archive"}) for res in results]
|
# return {
|
||||||
}
|
# "crawl_configs": [res.serialize(exclude={"archive"}) for res in results]
|
||||||
|
# }
|
||||||
|
|
||||||
@router.get("/{cid}")
|
@router.get("/{cid}")
|
||||||
async def get_crawl_config(crawl_config: CrawlConfig = Depends(crawls_dep)):
|
async def get_crawl_config(crawl_config: CrawlConfig = Depends(crawls_dep)):
|
||||||
|
Loading…
Reference in New Issue
Block a user