tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backend: add 'lastCrawlStartTime' and 'lastStartedByName' fields to crawlconfigs apis (#753)

Browse Source
This commit is contained in:
Ilya Kreymer 2023-04-17 08:34:29 -07:00 committed by GitHub
parent 59e49eacd5
commit 4a46f894a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 32 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
backend/btrixcloud/crawlconfigs.py
View File

@ -192,11 +192,13 @@ class CrawlConfigOut(CrawlConfig):
createdByName: Optional[str] createdByName: Optional[str]
modifiedByName: Optional[str] modifiedByName: Optional[str]
lastStartedByName: Optional[str]
firstSeed: Optional[str] firstSeed: Optional[str]
crawlCount: Optional[int] = 0 crawlCount: Optional[int] = 0
lastCrawlId: Optional[str] lastCrawlId: Optional[str]
lastCrawlStartTime: Optional[datetime]
lastCrawlTime: Optional[datetime] lastCrawlTime: Optional[datetime]
lastCrawlState: Optional[str] lastCrawlState: Optional[str]
@ -538,8 +540,29 @@ class CrawlConfigOps:
{"$unset": ["finishedCrawls"]}, {"$unset": ["finishedCrawls"]},
{"$set": {"lastCrawl": {"$arrayElemAt": ["$sortedCrawls", 0]}}}, {"$set": {"lastCrawl": {"$arrayElemAt": ["$sortedCrawls", 0]}}},
{"$set": {"lastCrawlId": "$lastCrawl._id"}}, {"$set": {"lastCrawlId": "$lastCrawl._id"}},
{"$set": {"lastCrawlStartTime": "$lastCrawl.started"}},
{"$set": {"lastCrawlTime": "$lastCrawl.finished"}}, {"$set": {"lastCrawlTime": "$lastCrawl.finished"}},
{"$set": {"lastCrawlState": "$lastCrawl.state"}}, {"$set": {"lastCrawlState": "$lastCrawl.state"}},
# Get userid of last started crawl
{"$set": {"lastStartedBy": "$lastCrawl.userid"}},
{
"$lookup": {
"from": "users",
"localField": "lastStartedBy",
"foreignField": "id",
"as": "lastStartedByName",
},
},
{
"$set": {
"lastStartedByName": {
"$arrayElemAt": ["$lastStartedByName.name", 0]
}
}
},
# total size
# {"$set": {"totalSize": {"$sum": "$$finishedCrawls.$$files.size"}}},
# unset
{"$unset": ["lastCrawl"]}, {"$unset": ["lastCrawl"]},
{"$unset": ["sortedCrawls"]}, {"$unset": ["sortedCrawls"]},
] ]
@ -655,7 +678,9 @@ class CrawlConfigOps:
) )
crawlconfig.crawlCount = crawl_stats["crawl_count"] crawlconfig.crawlCount = crawl_stats["crawl_count"]
crawlconfig.lastCrawlId = crawl_stats["last_crawl_id"] crawlconfig.lastCrawlId = crawl_stats["last_crawl_id"]
crawlconfig.lastCrawlStartTime = crawl_stats["last_crawl_started"]
crawlconfig.lastCrawlTime = crawl_stats["last_crawl_finished"] crawlconfig.lastCrawlTime = crawl_stats["last_crawl_finished"]
crawlconfig.lastStartedByName = crawl_stats["last_started_by"]
crawlconfig.lastCrawlState = crawl_stats["last_crawl_state"] crawlconfig.lastCrawlState = crawl_stats["last_crawl_state"]
return crawlconfig return crawlconfig

7
backend/btrixcloud/crawls.py
View File

@ -381,8 +381,10 @@ class CrawlOps:
stats = { stats = {
"crawl_count": 0, "crawl_count": 0,
"last_crawl_id": None, "last_crawl_id": None,
"last_crawl_started": None,
"last_crawl_finished": None, "last_crawl_finished": None,
"last_crawl_state": None, "last_crawl_state": None,
"last_started_by": None,
} }
match_query = {"cid": cid, "finished": {"$ne": None}, "inactive": {"$ne": True}} match_query = {"cid": cid, "finished": {"$ne": None}, "inactive": {"$ne": True}}
@ -393,9 +395,14 @@ class CrawlOps:
last_crawl = Crawl.from_dict(results[0]) last_crawl = Crawl.from_dict(results[0])
stats["last_crawl_id"] = str(last_crawl.id) stats["last_crawl_id"] = str(last_crawl.id)
stats["last_crawl_started"] = last_crawl.started
stats["last_crawl_finished"] = last_crawl.finished stats["last_crawl_finished"] = last_crawl.finished
stats["last_crawl_state"] = last_crawl.state stats["last_crawl_state"] = last_crawl.state
user = await self.user_manager.get(last_crawl.userid)
if user:
stats["last_started_by"] = user.name
return stats return stats
async def _resolve_crawl_refs( async def _resolve_crawl_refs(