From 172a9bf0cd83d894200f8885767109ad5be79050 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Tue, 9 Apr 2024 20:51:06 -0400 Subject: [PATCH] Change crawl.reviewStatus to 1-5 scale int (#1664) --- backend/btrixcloud/db.py | 2 +- .../migrations/migration_0009_crawl_types.py | 1 + .../migration_0026_crawl_review_status.py | 40 +++++++++++++++++++ backend/btrixcloud/models.py | 15 ++----- backend/test/test_run_crawl.py | 6 +-- .../pages/org/archived-item-detail/ui/qa.ts | 19 ++++++++- .../org/archived-item-qa/archived-item-qa.ts | 8 ++-- frontend/src/types/crawler.ts | 2 +- 8 files changed, 70 insertions(+), 23 deletions(-) create mode 100644 backend/btrixcloud/migrations/migration_0026_crawl_review_status.py diff --git a/backend/btrixcloud/db.py b/backend/btrixcloud/db.py index 93207ea1..2a13aa92 100644 --- a/backend/btrixcloud/db.py +++ b/backend/btrixcloud/db.py @@ -17,7 +17,7 @@ from pymongo.errors import InvalidName from .migrations import BaseMigration -CURR_DB_VERSION = "0025" +CURR_DB_VERSION = "0026" # ============================================================================ diff --git a/backend/btrixcloud/migrations/migration_0009_crawl_types.py b/backend/btrixcloud/migrations/migration_0009_crawl_types.py index 08e5bc60..dce2234d 100644 --- a/backend/btrixcloud/migrations/migration_0009_crawl_types.py +++ b/backend/btrixcloud/migrations/migration_0009_crawl_types.py @@ -20,6 +20,7 @@ class Migration(BaseMigration): Add type "crawl" to all existing crawls that don't already have a type """ + # pylint: disable=duplicate-code crawls = self.mdb["crawls"] try: await crawls.update_many( diff --git a/backend/btrixcloud/migrations/migration_0026_crawl_review_status.py b/backend/btrixcloud/migrations/migration_0026_crawl_review_status.py new file mode 100644 index 00000000..809f3e11 --- /dev/null +++ b/backend/btrixcloud/migrations/migration_0026_crawl_review_status.py @@ -0,0 +1,40 @@ +""" +Migration 0026 - Crawl reviewStatus type +""" + +from btrixcloud.migrations import BaseMigration + + +MIGRATION_VERSION = "0026" + + +class Migration(BaseMigration): + """Migration class.""" + + # pylint: disable=unused-argument + def __init__(self, mdb, **kwargs): + super().__init__(mdb, migration_version=MIGRATION_VERSION) + + async def migrate_up(self): + """Perform migration up. + + Switch crawl.reviewStatus from string to int between 1-5 + """ + # pylint: disable=duplicate-code + crawls = self.mdb["crawls"] + try: + await crawls.update_many( + {"reviewStatus": {"$eq": "good"}}, {"$set": {"reviewStatus": 5}} + ) + await crawls.update_many( + {"reviewStatus": {"$eq": "acceptable"}}, {"$set": {"reviewStatus": 3}} + ) + await crawls.update_many( + {"reviewStatus": {"$eq": "failure"}}, {"$set": {"reviewStatus": 1}} + ) + # pylint: disable=broad-exception-caught + except Exception as err: + print( + f"Error modifying existing crawl reviewStatuses to ints: {err}", + flush=True, + ) diff --git a/backend/btrixcloud/models.py b/backend/btrixcloud/models.py index 52191d9b..e0bd3f5e 100644 --- a/backend/btrixcloud/models.py +++ b/backend/btrixcloud/models.py @@ -535,15 +535,6 @@ class CrawlFileOut(BaseModel): expireAt: Optional[str] -# ============================================================================ -class ReviewStatus(str, Enum): - """QA review statuses""" - - GOOD = "good" - ACCEPTABLE = "acceptable" - FAILURE = "failure" - - # ============================================================================ class CrawlStats(BaseModel): """Crawl Stats for pages and size""" @@ -599,7 +590,7 @@ class BaseCrawl(CoreCrawlable, BaseMongoModel): collectionIds: Optional[List[UUID]] = [] - reviewStatus: Optional[ReviewStatus] = None + reviewStatus: Optional[conint(ge=1, le=5)] = None # type: ignore # ============================================================================ @@ -665,7 +656,7 @@ class CrawlOut(BaseMongoModel): crawlerChannel: str = "default" image: Optional[str] - reviewStatus: Optional[ReviewStatus] = None + reviewStatus: Optional[conint(ge=1, le=5)] = None # type: ignore # ============================================================================ @@ -684,7 +675,7 @@ class UpdateCrawl(BaseModel): description: Optional[str] tags: Optional[List[str]] collectionIds: Optional[List[UUID]] - reviewStatus: Optional[ReviewStatus] + reviewStatus: Optional[conint(ge=1, le=5)] # type: ignore # ============================================================================ diff --git a/backend/test/test_run_crawl.py b/backend/test/test_run_crawl.py index bcfa5ed0..0e8e9013 100644 --- a/backend/test/test_run_crawl.py +++ b/backend/test/test_run_crawl.py @@ -289,7 +289,7 @@ def test_update_crawl( f"{API_PREFIX}/orgs/{default_org_id}/crawls/{admin_crawl_id}", headers=admin_auth_headers, json={ - "reviewStatus": "good", + "reviewStatus": 5, }, ) assert r.status_code == 200 @@ -301,7 +301,7 @@ def test_update_crawl( headers=admin_auth_headers, ) assert r.status_code == 200 - assert r.json()["reviewStatus"] == "good" + assert r.json()["reviewStatus"] == 5 # Try to update to invalid reviewStatus r = requests.patch( @@ -318,7 +318,7 @@ def test_update_crawl( headers=admin_auth_headers, ) assert r.status_code == 200 - assert r.json()["reviewStatus"] == "good" + assert r.json()["reviewStatus"] == 5 # Verify deleting works as well r = requests.patch( diff --git a/frontend/src/pages/org/archived-item-detail/ui/qa.ts b/frontend/src/pages/org/archived-item-detail/ui/qa.ts index de7eaeda..db7ee5a3 100644 --- a/frontend/src/pages/org/archived-item-detail/ui/qa.ts +++ b/frontend/src/pages/org/archived-item-detail/ui/qa.ts @@ -53,6 +53,19 @@ function pageReviewStatus(page: ArchivedItemPage) { return msg("No review"); } +function crawlReviewStatus(status: number | null) { + if (status === 1) { + return msg("Failure"); + } + if (status === 3) { + return msg("Acceptable"); + } + if (status === 5) { + return msg("Good"); + } + return msg("No review"); +} + export function renderQA({ reviewStatus, qaCrawlExecSeconds, @@ -73,8 +86,10 @@ export function renderQA({ reviewStatus !== undefined, () => reviewStatus - ? html`${reviewStatus}` - : msg("None"), + ? html`${crawlReviewStatus(reviewStatus)}` + : msg("No review"), renderEmpty, )} diff --git a/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts b/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts index 47f57df6..f36c3534 100644 --- a/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts +++ b/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts @@ -494,10 +494,10 @@ export class ArchivedItemQA extends TailwindElement { class="mb-5" name="reviewStatus" label=${msg("Crawl quality assessment")} - value=${this.item?.reviewStatus ?? ""} + value=${this.item?.reviewStatus ?? null} required > - + ${msg("Failed")} - + ${msg("Acceptable")} - +