tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change crawl.reviewStatus to 1-5 scale int (#1664)

Browse Source
This commit is contained in:
Tessa Walsh 2024-04-09 20:51:06 -04:00 committed by GitHub
parent 9aba24a90e
commit 172a9bf0cd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 70 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/btrixcloud/db.py
View File

@ -17,7 +17,7 @@ from pymongo.errors import InvalidName
from .migrations import BaseMigration
CURR_DB_VERSION = "0025"
CURR_DB_VERSION = "0026"
# ============================================================================

1
backend/btrixcloud/migrations/migration_0009_crawl_types.py
View File

@ -20,6 +20,7 @@ class Migration(BaseMigration):
Add type "crawl" to all existing crawls that don't already have a type
"""
# pylint: disable=duplicate-code
crawls = self.mdb["crawls"]
try:
await crawls.update_many(

40
backend/btrixcloud/migrations/migration_0026_crawl_review_status.py Normal file
View File

@ -0,0 +1,40 @@
"""
Migration 0026 - Crawl reviewStatus type
"""
from btrixcloud.migrations import BaseMigration
MIGRATION_VERSION = "0026"
class Migration(BaseMigration):
"""Migration class."""
# pylint: disable=unused-argument
def __init__(self, mdb, **kwargs):
super().__init__(mdb, migration_version=MIGRATION_VERSION)
async def migrate_up(self):
"""Perform migration up.
Switch crawl.reviewStatus from string to int between 1-5
"""
# pylint: disable=duplicate-code
crawls = self.mdb["crawls"]
try:
await crawls.update_many(
{"reviewStatus": {"$eq": "good"}}, {"$set": {"reviewStatus": 5}}
)
await crawls.update_many(
{"reviewStatus": {"$eq": "acceptable"}}, {"$set": {"reviewStatus": 3}}
)
await crawls.update_many(
{"reviewStatus": {"$eq": "failure"}}, {"$set": {"reviewStatus": 1}}
)
# pylint: disable=broad-exception-caught
except Exception as err:
print(
f"Error modifying existing crawl reviewStatuses to ints: {err}",
flush=True,
)

15
backend/btrixcloud/models.py
View File

@ -535,15 +535,6 @@ class CrawlFileOut(BaseModel):
expireAt: Optional[str]
# ============================================================================
class ReviewStatus(str, Enum):
"""QA review statuses"""
GOOD = "good"
ACCEPTABLE = "acceptable"
FAILURE = "failure"
# ============================================================================
class CrawlStats(BaseModel):
"""Crawl Stats for pages and size"""
@ -599,7 +590,7 @@ class BaseCrawl(CoreCrawlable, BaseMongoModel):
collectionIds: Optional[List[UUID]] = []
reviewStatus: Optional[ReviewStatus] = None
reviewStatus: Optional[conint(ge=1, le=5)] = None # type: ignore
# ============================================================================
@ -665,7 +656,7 @@ class CrawlOut(BaseMongoModel):
crawlerChannel: str = "default"
image: Optional[str]
reviewStatus: Optional[ReviewStatus] = None
reviewStatus: Optional[conint(ge=1, le=5)] = None # type: ignore
# ============================================================================
@ -684,7 +675,7 @@ class UpdateCrawl(BaseModel):
description: Optional[str]
tags: Optional[List[str]]
collectionIds: Optional[List[UUID]]
reviewStatus: Optional[ReviewStatus]
reviewStatus: Optional[conint(ge=1, le=5)] # type: ignore
# ============================================================================

6
backend/test/test_run_crawl.py
View File

@ -289,7 +289,7 @@ def test_update_crawl(
f"{API_PREFIX}/orgs/{default_org_id}/crawls/{admin_crawl_id}",
headers=admin_auth_headers,
json={
"reviewStatus": "good",
"reviewStatus": 5,
},
)
assert r.status_code == 200
@ -301,7 +301,7 @@ def test_update_crawl(
headers=admin_auth_headers,
)
assert r.status_code == 200
assert r.json()["reviewStatus"] == "good"
assert r.json()["reviewStatus"] == 5
# Try to update to invalid reviewStatus
r = requests.patch(
@ -318,7 +318,7 @@ def test_update_crawl(
headers=admin_auth_headers,
)
assert r.status_code == 200
assert r.json()["reviewStatus"] == "good"
assert r.json()["reviewStatus"] == 5
# Verify deleting works as well
r = requests.patch(

19
frontend/src/pages/org/archived-item-detail/ui/qa.ts
View File

@ -53,6 +53,19 @@ function pageReviewStatus(page: ArchivedItemPage) {
return msg("No review");
}
function crawlReviewStatus(status: number | null) {
if (status === 1) {
return msg("Failure");
}
if (status === 3) {
return msg("Acceptable");
}
if (status === 5) {
return msg("Good");
}
return msg("No review");
}
export function renderQA({
reviewStatus,
qaCrawlExecSeconds,
@ -73,8 +86,10 @@ export function renderQA({
reviewStatus !== undefined,
() =>
reviewStatus
? html`<span class="capitalize">${reviewStatus}</span>`
: msg("None"),
? html`<span class="capitalize"
>${crawlReviewStatus(reviewStatus)}</span
>`
: msg("No review"),
renderEmpty,
)}
</btrix-desc-list-item>

8
frontend/src/pages/org/archived-item-qa/archived-item-qa.ts
View File

@ -494,10 +494,10 @@ export class ArchivedItemQA extends TailwindElement {
class="mb-5"
name="reviewStatus"
label=${msg("Crawl quality assessment")}
value=${this.item?.reviewStatus ?? ""}
value=${this.item?.reviewStatus ?? null}
required
>
<sl-radio-button value="failure">
<sl-radio-button value="1">
<sl-icon
name="patch-exclamation"
slot="prefix"
@ -505,7 +505,7 @@ export class ArchivedItemQA extends TailwindElement {
></sl-icon>
${msg("Failed")}
</sl-radio-button>
<sl-radio-button value="acceptable" checked>
<sl-radio-button value="3" checked>
<sl-icon
name="patch-minus"
slot="prefix"
@ -513,7 +513,7 @@ export class ArchivedItemQA extends TailwindElement {
></sl-icon>
${msg("Acceptable")}
</sl-radio-button>
<sl-radio-button value="good">
<sl-radio-button value="5">
<sl-icon
name="patch-check"
slot="prefix"

2
frontend/src/types/crawler.ts
View File

@ -147,7 +147,7 @@ type ArchivedItemBase = {
tags: string[];
crawlExecSeconds: number;
qaCrawlExecSeconds: number;
reviewStatus: "good" | "acceptable" | "failure" | null;
reviewStatus?: number;
};
export type Crawl = ArchivedItemBase &