From cbcf087a48b93b6abd393fa16cd36b1000bc4583 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Tue, 14 Jan 2025 10:57:06 -0500 Subject: [PATCH] Add last crawl and subscription status indicators to org list (#2273) Fixes #2260 - Adds `lastCrawlFinished` to Organization model, updated after crawls are added/deleted and with an idempotent migration to backfill existing orgs - Adds Last Crawl column to end of admin orgs list table - Adds subscription icon next to existing status icon in orgs list - Adds "lastCrawlFinished", "subscriptionStatus", and "subscriptionPlan" sort options to orgs list backend endpoint in anticipation of future sorting/filtering of orgs list --------- Co-authored-by: emma Co-authored-by: Henry Wilkinson Co-authored-by: Ilya Kreymer --- backend/btrixcloud/basecrawls.py | 19 ++++- backend/btrixcloud/db.py | 15 +++- .../migration_0038_org_last_crawl_finished.py | 43 ++++++++++ backend/btrixcloud/models.py | 4 + backend/btrixcloud/operator/crawls.py | 1 + backend/btrixcloud/orgs.py | 23 +++++- backend/test/test_org.py | 20 +++++ frontend/src/components/orgs-list.ts | 80 ++++++++++++++++++- .../features/collections/share-collection.ts | 2 +- frontend/src/types/org.ts | 1 + 10 files changed, 198 insertions(+), 10 deletions(-) create mode 100644 backend/btrixcloud/migrations/migration_0038_org_last_crawl_finished.py diff --git a/backend/btrixcloud/basecrawls.py b/backend/btrixcloud/basecrawls.py index 01e700f8..99b06597 100644 --- a/backend/btrixcloud/basecrawls.py +++ b/backend/btrixcloud/basecrawls.py @@ -1,6 +1,6 @@ """ base crawl type """ -from datetime import timedelta +from datetime import datetime, timedelta from typing import Optional, List, Union, Dict, Any, Type, TYPE_CHECKING, cast, Tuple from uuid import UUID import urllib.parse @@ -359,6 +359,8 @@ class BaseCrawlOps: await self.orgs.inc_org_bytes_stored(org.id, -size, type_) + await self.orgs.set_last_crawl_finished(org.id) + quota_reached = self.orgs.storage_quota_reached(org) return res.deleted_count, cids_to_update, quota_reached @@ -853,6 +855,21 @@ class BaseCrawlOps: return total_size, crawls_size, uploads_size + async def get_org_last_crawl_finished(self, oid: UUID) -> Optional[datetime]: + """Get last crawl finished time for org""" + last_crawl_finished: Optional[datetime] = None + + cursor = ( + self.crawls.find({"oid": oid, "finished": {"$ne": None}}) + .sort({"finished": -1}) + .limit(1) + ) + last_crawl = await cursor.to_list(length=1) + if last_crawl: + last_crawl_finished = last_crawl[0].get("finished") + + return last_crawl_finished + # ============================================================================ def init_base_crawls_api(app, user_dep, *args): diff --git a/backend/btrixcloud/db.py b/backend/btrixcloud/db.py index 0723258e..9f0bab04 100644 --- a/backend/btrixcloud/db.py +++ b/backend/btrixcloud/db.py @@ -17,7 +17,7 @@ from pymongo.errors import InvalidName from .migrations import BaseMigration -CURR_DB_VERSION = "0037" +CURR_DB_VERSION = "0038" # ============================================================================ @@ -95,8 +95,11 @@ async def update_and_prepare_db( """ await ping_db(mdb) print("Database setup started", flush=True) - if await run_db_migrations(mdb, user_manager, background_job_ops, page_ops): + if await run_db_migrations( + mdb, user_manager, page_ops, org_ops, background_job_ops + ): await drop_indexes(mdb) + await create_indexes( org_ops, crawl_ops, @@ -114,7 +117,8 @@ async def update_and_prepare_db( # ============================================================================ -async def run_db_migrations(mdb, user_manager, background_job_ops, page_ops): +# pylint: disable=too-many-locals +async def run_db_migrations(mdb, user_manager, page_ops, org_ops, background_job_ops): """Run database migrations.""" # if first run, just set version and exit @@ -147,7 +151,10 @@ async def run_db_migrations(mdb, user_manager, background_job_ops, page_ops): migration_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(migration_module) migration = migration_module.Migration( - mdb, background_job_ops=background_job_ops, page_ops=page_ops + mdb, + page_ops=page_ops, + org_ops=org_ops, + background_job_ops=background_job_ops, ) if await migration.run(): migrations_run = True diff --git a/backend/btrixcloud/migrations/migration_0038_org_last_crawl_finished.py b/backend/btrixcloud/migrations/migration_0038_org_last_crawl_finished.py new file mode 100644 index 00000000..7b0b146b --- /dev/null +++ b/backend/btrixcloud/migrations/migration_0038_org_last_crawl_finished.py @@ -0,0 +1,43 @@ +""" +Migration 0038 - Organization lastCrawlFinished field +""" + +from btrixcloud.migrations import BaseMigration + + +MIGRATION_VERSION = "0038" + + +class Migration(BaseMigration): + """Migration class.""" + + # pylint: disable=unused-argument + def __init__(self, mdb, **kwargs): + super().__init__(mdb, migration_version=MIGRATION_VERSION) + + self.org_ops = kwargs.get("org_ops") + + async def migrate_up(self): + """Perform migration up. Set lastCrawlFinished for each org.""" + # pylint: disable=duplicate-code, line-too-long + if self.org_ops is None: + print( + "Unable to set lastCrawlFinished for orgs, missing org_ops", flush=True + ) + return + + orgs_db = self.mdb["organizations"] + async for org_dict in orgs_db.find({}): + oid = org_dict.get("_id") + + if org_dict.get("lastCrawlFinished"): + continue + + try: + await self.org_ops.set_last_crawl_finished(oid) + # pylint: disable=broad-exception-caught + except Exception as err: + print( + f"Error setting lastCrawlFinished for org {oid}: {err}", + flush=True, + ) diff --git a/backend/btrixcloud/models.py b/backend/btrixcloud/models.py index 807628fd..31a6e475 100644 --- a/backend/btrixcloud/models.py +++ b/backend/btrixcloud/models.py @@ -1723,6 +1723,8 @@ class OrgOut(BaseMongoModel): allowedProxies: list[str] = [] crawlingDefaults: Optional[CrawlConfigDefaults] = None + lastCrawlFinished: Optional[datetime] = None + enablePublicProfile: bool = False publicDescription: str = "" publicUrl: str = "" @@ -1782,6 +1784,8 @@ class Organization(BaseMongoModel): allowedProxies: list[str] = [] crawlingDefaults: Optional[CrawlConfigDefaults] = None + lastCrawlFinished: Optional[datetime] = None + enablePublicProfile: bool = False publicDescription: Optional[str] = None publicUrl: Optional[str] = None diff --git a/backend/btrixcloud/operator/crawls.py b/backend/btrixcloud/operator/crawls.py index a39064a1..737397fb 100644 --- a/backend/btrixcloud/operator/crawls.py +++ b/backend/btrixcloud/operator/crawls.py @@ -1506,6 +1506,7 @@ class CrawlOperator(BaseOperator): await self.org_ops.inc_org_bytes_stored( crawl.oid, status.filesAddedSize, "crawl" ) + await self.org_ops.set_last_crawl_finished(crawl.oid) await self.coll_ops.add_successful_crawl_to_collections(crawl.id, crawl.cid) if state in FAILED_STATES: diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 8ba4d350..d0ca47b2 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -221,12 +221,25 @@ class OrgOps: sort_query = {"default": -1} if sort_by: - sort_fields = ("name", "slug", "readOnly") + sort_fields = ( + "name", + "slug", + "readOnly", + "lastCrawlFinished", + "subscriptionStatus", + "subscriptionPlan", + ) if sort_by not in sort_fields: raise HTTPException(status_code=400, detail="invalid_sort_by") if sort_direction not in (1, -1): raise HTTPException(status_code=400, detail="invalid_sort_direction") + if sort_by == "subscriptionStatus": + sort_by = "subscription.status" + + if sort_by == "subscriptionPlan": + sort_by = "subscription.planId" + # Do lexical sort of names if sort_by == "name": sort_by = "nameLower" @@ -1382,6 +1395,14 @@ class OrgOps: return {"success": True} + async def set_last_crawl_finished(self, oid: UUID): + """Recalculate and set lastCrawlFinished field on org""" + last_crawl_finished = await self.base_crawl_ops.get_org_last_crawl_finished(oid) + await self.orgs.find_one_and_update( + {"_id": oid}, + {"$set": {"lastCrawlFinished": last_crawl_finished}}, + ) + # ============================================================================ # pylint: disable=too-many-statements, too-many-arguments diff --git a/backend/test/test_org.py b/backend/test/test_org.py index a3de79c5..57c0b8fc 100644 --- a/backend/test/test_org.py +++ b/backend/test/test_org.py @@ -755,3 +755,23 @@ def test_sort_orgs(admin_auth_headers): if last_name: assert org_name_lower < last_name last_name = org_name_lower + + # Sort desc by lastCrawlFinished, ensure default org still first + r = requests.get( + f"{API_PREFIX}/orgs?sortBy=lastCrawlFinished&sortDirection=-1", + headers=admin_auth_headers, + ) + data = r.json() + orgs = data["items"] + + assert orgs[0]["default"] + + other_orgs = orgs[1:] + last_last_crawl_finished = None + for org in other_orgs: + last_crawl_finished = org.get("lastCrawlFinished") + if not last_crawl_finished: + continue + if last_last_crawl_finished: + assert last_crawl_finished <= last_last_crawl_finished + last_last_crawl_finished = last_crawl_finished diff --git a/frontend/src/components/orgs-list.ts b/frontend/src/components/orgs-list.ts index 37f879a0..82d4279f 100644 --- a/frontend/src/components/orgs-list.ts +++ b/frontend/src/components/orgs-list.ts @@ -14,6 +14,7 @@ import { when } from "lit/directives/when.js"; import { BtrixElement } from "@/classes/BtrixElement"; import type { Dialog } from "@/components/ui/dialog"; +import { SubscriptionStatus } from "@/types/billing"; import type { ProxiesAPIResponse, Proxy } from "@/types/crawler"; import type { OrgData } from "@/utils/orgs"; @@ -26,7 +27,7 @@ import type { OrgData } from "@/utils/orgs"; export class OrgsList extends BtrixElement { static styles = css` btrix-table { - grid-template-columns: min-content [clickable-start] 50ch auto auto auto [clickable-end] min-content; + grid-template-columns: min-content [clickable-start] minmax(auto, 50ch) auto auto auto [clickable-end] min-content; } `; @@ -130,6 +131,9 @@ export class OrgsList extends BtrixElement { ${msg("Bytes Stored")} + + ${msg("Last Crawl")} + ${msg("Actions")} @@ -644,16 +648,82 @@ export class OrgsList extends BtrixElement { }; } + let subscription = { + icon: none, + description: msg("No Subscription"), + }; + + if (org.subscription) { + switch (org.subscription.status) { + case SubscriptionStatus.Active: + subscription = { + icon: html``, + description: msg("Active Subscription"), + }; + break; + case SubscriptionStatus.Trialing: + subscription = { + icon: html``, + description: msg("Trial"), + }; + break; + case SubscriptionStatus.TrialingCanceled: + subscription = { + icon: html``, + description: msg("Trial Canceled"), + }; + break; + case SubscriptionStatus.PausedPaymentFailed: + subscription = { + icon: html``, + description: msg("Payment Failed"), + }; + break; + case SubscriptionStatus.Cancelled: + subscription = { + icon: html` + `, + description: msg("Canceled"), + }; + break; + default: + break; + } + } + return html` - + ${status.icon} + + ${subscription.icon} + - ${this.localize.date(org.created, { dateStyle: "short" })} @@ -682,6 +751,11 @@ export class OrgsList extends BtrixElement { ? this.localize.bytes(org.bytesStored, { unitDisplay: "narrow" }) : none} + + ${org.lastCrawlFinished + ? this.localize.date(org.lastCrawlFinished, { dateStyle: "short" }) + : none} + e.stopPropagation()} diff --git a/frontend/src/features/collections/share-collection.ts b/frontend/src/features/collections/share-collection.ts index 77252091..ea9fe449 100644 --- a/frontend/src/features/collections/share-collection.ts +++ b/frontend/src/features/collections/share-collection.ts @@ -242,7 +242,7 @@ export class ShareCollection extends BtrixElement { @sl-after-hide=${() => { this.tabGroup?.show(Tab.Link); }} - class="[--body-spacing:0] [--width:40rem]" + class="[--width:40rem] [--body-spacing:0]" >