tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backend: fix crawl config revision links (#149)

Browse Source 
backed: crawlconfig:
- ensure newId is saved on old config being replaced
- if old config replaced is being deleted, ensure newId link is set on its old config (if any),
and the oldId points to the oldId of config being replaced (if any)
This commit is contained in:
Ilya Kreymer 2022-02-21 16:51:27 -08:00 committed by GitHub
parent f30b398fea
commit aa5207915c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
backend/crawlconfigs.py
View File

@ -203,7 +203,12 @@ class CrawlConfigOps:
old_config = await self.get_crawl_config(old_id, archive) old_config = await self.get_crawl_config(old_id, archive)
async with await self.dbclient.start_session() as sesh: async with await self.dbclient.start_session() as sesh:
async with sesh.start_transaction(): async with sesh.start_transaction():
await self.make_inactive(old_config, data["_id"]) if (
await self.make_inactive_or_delete(old_config, data["_id"])
== "deleted"
):
data["oldId"] = old_config.oldId
result = await self.crawl_configs.insert_one(data) result = await self.crawl_configs.insert_one(data)
else: else:
@ -221,7 +226,9 @@ class CrawlConfigOps:
""" Update name, scale and/or schedule for an existing crawl config """ """ Update name, scale and/or schedule for an existing crawl config """
# set update query # set update query
query = update.dict(exclude_unset=True, exclude_defaults=True, exclude_none=True) query = update.dict(
exclude_unset=True, exclude_defaults=True, exclude_none=True
)
if len(query) == 0: if len(query) == 0:
raise HTTPException(status_code=400, detail="no_update_data") raise HTTPException(status_code=400, detail="no_update_data")
@ -343,11 +350,15 @@ class CrawlConfigOps:
res = await self.crawl_configs.find_one(query) res = await self.crawl_configs.find_one(query)
return config_cls.from_dict(res) return config_cls.from_dict(res)
async def make_inactive(self, crawlconfig: CrawlConfig, new_id: uuid.UUID = None): async def make_inactive_or_delete(
"""Delete config, if no crawls ran yet. Otherwise, move to inactive list""" self, crawlconfig: CrawlConfig, new_id: uuid.UUID = None
):
"""Make config inactive if crawls exist, otherwise move to inactive list"""
query = {"inactive": True}
if new_id: if new_id:
crawlconfig.newId = new_id crawlconfig.newId = query["newId"] = new_id
if await self.get_running_crawl(crawlconfig): if await self.get_running_crawl(crawlconfig):
raise HTTPException(status_code=400, detail="crawl_running_cant_deactivate") raise HTTPException(status_code=400, detail="crawl_running_cant_deactivate")
@ -364,13 +375,18 @@ class CrawlConfigOps:
if result.deleted_count != 1: if result.deleted_count != 1:
raise HTTPException(status_code=404, detail="failed_to_delete") raise HTTPException(status_code=404, detail="failed_to_delete")
if crawlconfig.oldId:
await self.crawl_configs.find_one_and_update(
{"_id": crawlconfig.oldId}, {"$set": query}
)
status = "deleted" status = "deleted"
else: else:
if not await self.crawl_configs.find_one_and_update( if not await self.crawl_configs.find_one_and_update(
{"_id": crawlconfig.id, "inactive": {"$ne": True}}, {"_id": crawlconfig.id, "inactive": {"$ne": True}},
{"$set": {"inactive": True}}, {"$set": query},
): ):
raise HTTPException(status_code=404, detail="failed_to_deactivate") raise HTTPException(status_code=404, detail="failed_to_deactivate")
@ -386,7 +402,7 @@ class CrawlConfigOps:
async with await self.dbclient.start_session() as sesh: async with await self.dbclient.start_session() as sesh:
async with sesh.start_transaction(): async with sesh.start_transaction():
status = await self.make_inactive(crawlconfig) status = await self.make_inactive_or_delete(crawlconfig)
return {"success": True, "status": status} return {"success": True, "status": status}