Change crawl.reviewStatus to 1-5 scale int (#1664)

backend/btrixcloud/db.py

@@ -17,7 +17,7 @@ from pymongo.errors import InvalidName
 from .migrations import BaseMigration
 
 
-CURR_DB_VERSION = "0025"
+CURR_DB_VERSION = "0026"
 
 
 # ============================================================================

backend/btrixcloud/migrations/migration_0009_crawl_types.py

@@ -20,6 +20,7 @@ class Migration(BaseMigration):
 
         Add type "crawl" to all existing crawls that don't already have a type
         """
+        # pylint: disable=duplicate-code
         crawls = self.mdb["crawls"]
         try:
             await crawls.update_many(

backend/btrixcloud/migrations/migration_0026_crawl_review_status.py

@@ -0,0 +1,40 @@
+"""
+Migration 0026 - Crawl reviewStatus type
+"""
+
+from btrixcloud.migrations import BaseMigration
+
+
+MIGRATION_VERSION = "0026"
+
+
+class Migration(BaseMigration):
+    """Migration class."""
+
+    # pylint: disable=unused-argument
+    def __init__(self, mdb, **kwargs):
+        super().__init__(mdb, migration_version=MIGRATION_VERSION)
+
+    async def migrate_up(self):
+        """Perform migration up.
+
+        Switch crawl.reviewStatus from string to int between 1-5
+        """
+        # pylint: disable=duplicate-code
+        crawls = self.mdb["crawls"]
+        try:
+            await crawls.update_many(
+                {"reviewStatus": {"$eq": "good"}}, {"$set": {"reviewStatus": 5}}
+            )
+            await crawls.update_many(
+                {"reviewStatus": {"$eq": "acceptable"}}, {"$set": {"reviewStatus": 3}}
+            )
+            await crawls.update_many(
+                {"reviewStatus": {"$eq": "failure"}}, {"$set": {"reviewStatus": 1}}
+            )
+        # pylint: disable=broad-exception-caught
+        except Exception as err:
+            print(
+                f"Error modifying existing crawl reviewStatuses to ints: {err}",
+                flush=True,
+            )

backend/btrixcloud/models.py

@@ -535,15 +535,6 @@ class CrawlFileOut(BaseModel):
     expireAt: Optional[str]
 
 
-# ============================================================================
-class ReviewStatus(str, Enum):
-    """QA review statuses"""
-
-    GOOD = "good"
-    ACCEPTABLE = "acceptable"
-    FAILURE = "failure"
-
-
 # ============================================================================
 class CrawlStats(BaseModel):
     """Crawl Stats for pages and size"""
@@ -599,7 +590,7 @@ class BaseCrawl(CoreCrawlable, BaseMongoModel):
 
     collectionIds: Optional[List[UUID]] = []
 
-    reviewStatus: Optional[ReviewStatus] = None
+    reviewStatus: Optional[conint(ge=1, le=5)] = None  # type: ignore
 
 
 # ============================================================================
@@ -665,7 +656,7 @@ class CrawlOut(BaseMongoModel):
     crawlerChannel: str = "default"
     image: Optional[str]
 
-    reviewStatus: Optional[ReviewStatus] = None
+    reviewStatus: Optional[conint(ge=1, le=5)] = None  # type: ignore
 
 
 # ============================================================================
@@ -684,7 +675,7 @@ class UpdateCrawl(BaseModel):
     description: Optional[str]
     tags: Optional[List[str]]
     collectionIds: Optional[List[UUID]]
-    reviewStatus: Optional[ReviewStatus]
+    reviewStatus: Optional[conint(ge=1, le=5)]  # type: ignore
 
 
 # ============================================================================

backend/test/test_run_crawl.py

@@ -289,7 +289,7 @@ def test_update_crawl(
         f"{API_PREFIX}/orgs/{default_org_id}/crawls/{admin_crawl_id}",
         headers=admin_auth_headers,
         json={
-            "reviewStatus": "good",
+            "reviewStatus": 5,
         },
     )
     assert r.status_code == 200
@@ -301,7 +301,7 @@ def test_update_crawl(
         headers=admin_auth_headers,
     )
     assert r.status_code == 200
-    assert r.json()["reviewStatus"] == "good"
+    assert r.json()["reviewStatus"] == 5
 
     # Try to update to invalid reviewStatus
     r = requests.patch(
@@ -318,7 +318,7 @@ def test_update_crawl(
         headers=admin_auth_headers,
     )
     assert r.status_code == 200
-    assert r.json()["reviewStatus"] == "good"
+    assert r.json()["reviewStatus"] == 5
 
     # Verify deleting works as well
     r = requests.patch(

frontend/src/pages/org/archived-item-detail/ui/qa.ts

@@ -53,6 +53,19 @@ function pageReviewStatus(page: ArchivedItemPage) {
   return msg("No review");
 }
 
+function crawlReviewStatus(status: number | null) {
+  if (status === 1) {
+    return msg("Failure");
+  }
+  if (status === 3) {
+    return msg("Acceptable");
+  }
+  if (status === 5) {
+    return msg("Good");
+  }
+  return msg("No review");
+}
+
 export function renderQA({
   reviewStatus,
   qaCrawlExecSeconds,
@@ -73,8 +86,10 @@ export function renderQA({
             reviewStatus !== undefined,
             () =>
               reviewStatus
-                ? html`<span class="capitalize">${reviewStatus}</span>`
+                ? html`<span class="capitalize"
-                : msg("None"),
+                    >${crawlReviewStatus(reviewStatus)}</span
+                  >`
+                : msg("No review"),
             renderEmpty,
           )}
         </btrix-desc-list-item>

frontend/src/pages/org/archived-item-qa/archived-item-qa.ts

@@ -494,10 +494,10 @@ export class ArchivedItemQA extends TailwindElement {
             class="mb-5"
             name="reviewStatus"
             label=${msg("Crawl quality assessment")}
-            value=${this.item?.reviewStatus ?? ""}
+            value=${this.item?.reviewStatus ?? null}
             required
           >
-            <sl-radio-button value="failure">
+            <sl-radio-button value="1">
               <sl-icon
                 name="patch-exclamation"
                 slot="prefix"
@@ -505,7 +505,7 @@ export class ArchivedItemQA extends TailwindElement {
               ></sl-icon>
               ${msg("Failed")}
             </sl-radio-button>
-            <sl-radio-button value="acceptable" checked>
+            <sl-radio-button value="3" checked>
               <sl-icon
                 name="patch-minus"
                 slot="prefix"
@@ -513,7 +513,7 @@ export class ArchivedItemQA extends TailwindElement {
               ></sl-icon>
               ${msg("Acceptable")}
             </sl-radio-button>
-            <sl-radio-button value="good">
+            <sl-radio-button value="5">
               <sl-icon
                 name="patch-check"
                 slot="prefix"

frontend/src/types/crawler.ts

@@ -147,7 +147,7 @@ type ArchivedItemBase = {
   tags: string[];
   crawlExecSeconds: number;
   qaCrawlExecSeconds: number;
-  reviewStatus: "good" | "acceptable" | "failure" | null;
+  reviewStatus?: number;
 };
 
 export type Crawl = ArchivedItemBase &