backend api: add curr crawl to crawlconfig listing (#95)
* backend api: add current crawl id to crawlconfig listing - model: add 'currCrawlId' to CrawlConfig model - output: add response model to /crawlconfigs api response to show correct openapi model - rename crawl_configs -> crawlConfigs for consistency
This commit is contained in:
parent
ec1a758e42
commit
b506442b21
@ -110,6 +110,19 @@ class CrawlConfig(BaseMongoModel):
|
||||
lastCrawlTime: Optional[datetime]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
class CrawlConfigOut(CrawlConfig):
|
||||
"""Crawl Config Output, includes currCrawlId of running crawl"""
|
||||
|
||||
currCrawlId: Optional[str]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
class CrawlConfigsResponse(BaseModel):
|
||||
""" model for crawl configs response """
|
||||
crawlConfigs: List[CrawlConfigOut]
|
||||
|
||||
|
||||
# ============================================================================
|
||||
class UpdateSchedule(BaseModel):
|
||||
""" Update the crawl schedule """
|
||||
@ -187,7 +200,21 @@ class CrawlOps:
|
||||
"""Get all crawl configs for an archive is a member of"""
|
||||
cursor = self.crawl_configs.find({"archive": archive.id})
|
||||
results = await cursor.to_list(length=1000)
|
||||
return [CrawlConfig.from_dict(res) for res in results]
|
||||
|
||||
crawls = await self.crawl_manager.list_running_crawls(aid=archive.id)
|
||||
|
||||
running = {}
|
||||
for crawl in crawls:
|
||||
running[crawl.cid] = crawl.id
|
||||
|
||||
configs = []
|
||||
for res in results:
|
||||
config = CrawlConfigOut.from_dict(res)
|
||||
# pylint: disable=invalid-name
|
||||
config.currCrawlId = running.get(config.id)
|
||||
configs.append(config)
|
||||
|
||||
return configs
|
||||
|
||||
async def get_crawl_config(self, cid: str, archive: Archive):
|
||||
"""Get an archive for user by unique id"""
|
||||
@ -226,12 +253,13 @@ def init_crawl_config_api(mdb, user_dep, archive_ops, crawl_manager):
|
||||
|
||||
return archive
|
||||
|
||||
@router.get("")
|
||||
@router.get("", response_model=CrawlConfigsResponse)
|
||||
async def get_crawl_configs(archive: Archive = Depends(archive_crawl_dep)):
|
||||
results = await ops.get_crawl_configs(archive)
|
||||
return {
|
||||
"crawl_configs": [res.serialize(exclude={"archive"}) for res in results]
|
||||
}
|
||||
return CrawlConfigsResponse(crawlConfigs=results)
|
||||
# return {
|
||||
# "crawl_configs": [res.serialize(exclude={"archive"}) for res in results]
|
||||
# }
|
||||
|
||||
@router.get("/{cid}")
|
||||
async def get_crawl_config(crawl_config: CrawlConfig = Depends(crawls_dep)):
|
||||
|
Loading…
Reference in New Issue
Block a user