Add last crawl and subscription status indicators to org list (#2273)
Fixes #2260 - Adds `lastCrawlFinished` to Organization model, updated after crawls are added/deleted and with an idempotent migration to backfill existing orgs - Adds Last Crawl column to end of admin orgs list table - Adds subscription icon next to existing status icon in orgs list - Adds "lastCrawlFinished", "subscriptionStatus", and "subscriptionPlan" sort options to orgs list backend endpoint in anticipation of future sorting/filtering of orgs list --------- Co-authored-by: emma <hi@emma.cafe> Co-authored-by: Henry Wilkinson <henry@wilkinson.graphics> Co-authored-by: Ilya Kreymer <ikreymer@gmail.com>
This commit is contained in:
		
							parent
							
								
									04e9127d35
								
							
						
					
					
						commit
						cbcf087a48
					
				| @ -1,6 +1,6 @@ | ||||
| """ base crawl type """ | ||||
| 
 | ||||
| from datetime import timedelta | ||||
| from datetime import datetime, timedelta | ||||
| from typing import Optional, List, Union, Dict, Any, Type, TYPE_CHECKING, cast, Tuple | ||||
| from uuid import UUID | ||||
| import urllib.parse | ||||
| @ -359,6 +359,8 @@ class BaseCrawlOps: | ||||
| 
 | ||||
|         await self.orgs.inc_org_bytes_stored(org.id, -size, type_) | ||||
| 
 | ||||
|         await self.orgs.set_last_crawl_finished(org.id) | ||||
| 
 | ||||
|         quota_reached = self.orgs.storage_quota_reached(org) | ||||
| 
 | ||||
|         return res.deleted_count, cids_to_update, quota_reached | ||||
| @ -853,6 +855,21 @@ class BaseCrawlOps: | ||||
| 
 | ||||
|         return total_size, crawls_size, uploads_size | ||||
| 
 | ||||
|     async def get_org_last_crawl_finished(self, oid: UUID) -> Optional[datetime]: | ||||
|         """Get last crawl finished time for org""" | ||||
|         last_crawl_finished: Optional[datetime] = None | ||||
| 
 | ||||
|         cursor = ( | ||||
|             self.crawls.find({"oid": oid, "finished": {"$ne": None}}) | ||||
|             .sort({"finished": -1}) | ||||
|             .limit(1) | ||||
|         ) | ||||
|         last_crawl = await cursor.to_list(length=1) | ||||
|         if last_crawl: | ||||
|             last_crawl_finished = last_crawl[0].get("finished") | ||||
| 
 | ||||
|         return last_crawl_finished | ||||
| 
 | ||||
| 
 | ||||
| # ============================================================================ | ||||
| def init_base_crawls_api(app, user_dep, *args): | ||||
|  | ||||
| @ -17,7 +17,7 @@ from pymongo.errors import InvalidName | ||||
| from .migrations import BaseMigration | ||||
| 
 | ||||
| 
 | ||||
| CURR_DB_VERSION = "0037" | ||||
| CURR_DB_VERSION = "0038" | ||||
| 
 | ||||
| 
 | ||||
| # ============================================================================ | ||||
| @ -95,8 +95,11 @@ async def update_and_prepare_db( | ||||
|     """ | ||||
|     await ping_db(mdb) | ||||
|     print("Database setup started", flush=True) | ||||
|     if await run_db_migrations(mdb, user_manager, background_job_ops, page_ops): | ||||
|     if await run_db_migrations( | ||||
|         mdb, user_manager, page_ops, org_ops, background_job_ops | ||||
|     ): | ||||
|         await drop_indexes(mdb) | ||||
| 
 | ||||
|     await create_indexes( | ||||
|         org_ops, | ||||
|         crawl_ops, | ||||
| @ -114,7 +117,8 @@ async def update_and_prepare_db( | ||||
| 
 | ||||
| 
 | ||||
| # ============================================================================ | ||||
| async def run_db_migrations(mdb, user_manager, background_job_ops, page_ops): | ||||
| # pylint: disable=too-many-locals | ||||
| async def run_db_migrations(mdb, user_manager, page_ops, org_ops, background_job_ops): | ||||
|     """Run database migrations.""" | ||||
| 
 | ||||
|     # if first run, just set version and exit | ||||
| @ -147,7 +151,10 @@ async def run_db_migrations(mdb, user_manager, background_job_ops, page_ops): | ||||
|             migration_module = importlib.util.module_from_spec(spec) | ||||
|             spec.loader.exec_module(migration_module) | ||||
|             migration = migration_module.Migration( | ||||
|                 mdb, background_job_ops=background_job_ops, page_ops=page_ops | ||||
|                 mdb, | ||||
|                 page_ops=page_ops, | ||||
|                 org_ops=org_ops, | ||||
|                 background_job_ops=background_job_ops, | ||||
|             ) | ||||
|             if await migration.run(): | ||||
|                 migrations_run = True | ||||
|  | ||||
| @ -0,0 +1,43 @@ | ||||
| """ | ||||
| Migration 0038 - Organization lastCrawlFinished field | ||||
| """ | ||||
| 
 | ||||
| from btrixcloud.migrations import BaseMigration | ||||
| 
 | ||||
| 
 | ||||
| MIGRATION_VERSION = "0038" | ||||
| 
 | ||||
| 
 | ||||
| class Migration(BaseMigration): | ||||
|     """Migration class.""" | ||||
| 
 | ||||
|     # pylint: disable=unused-argument | ||||
|     def __init__(self, mdb, **kwargs): | ||||
|         super().__init__(mdb, migration_version=MIGRATION_VERSION) | ||||
| 
 | ||||
|         self.org_ops = kwargs.get("org_ops") | ||||
| 
 | ||||
|     async def migrate_up(self): | ||||
|         """Perform migration up. Set lastCrawlFinished for each org.""" | ||||
|         # pylint: disable=duplicate-code, line-too-long | ||||
|         if self.org_ops is None: | ||||
|             print( | ||||
|                 "Unable to set lastCrawlFinished for orgs, missing org_ops", flush=True | ||||
|             ) | ||||
|             return | ||||
| 
 | ||||
|         orgs_db = self.mdb["organizations"] | ||||
|         async for org_dict in orgs_db.find({}): | ||||
|             oid = org_dict.get("_id") | ||||
| 
 | ||||
|             if org_dict.get("lastCrawlFinished"): | ||||
|                 continue | ||||
| 
 | ||||
|             try: | ||||
|                 await self.org_ops.set_last_crawl_finished(oid) | ||||
|             # pylint: disable=broad-exception-caught | ||||
|             except Exception as err: | ||||
|                 print( | ||||
|                     f"Error setting lastCrawlFinished for org {oid}: {err}", | ||||
|                     flush=True, | ||||
|                 ) | ||||
| @ -1723,6 +1723,8 @@ class OrgOut(BaseMongoModel): | ||||
|     allowedProxies: list[str] = [] | ||||
|     crawlingDefaults: Optional[CrawlConfigDefaults] = None | ||||
| 
 | ||||
|     lastCrawlFinished: Optional[datetime] = None | ||||
| 
 | ||||
|     enablePublicProfile: bool = False | ||||
|     publicDescription: str = "" | ||||
|     publicUrl: str = "" | ||||
| @ -1782,6 +1784,8 @@ class Organization(BaseMongoModel): | ||||
|     allowedProxies: list[str] = [] | ||||
|     crawlingDefaults: Optional[CrawlConfigDefaults] = None | ||||
| 
 | ||||
|     lastCrawlFinished: Optional[datetime] = None | ||||
| 
 | ||||
|     enablePublicProfile: bool = False | ||||
|     publicDescription: Optional[str] = None | ||||
|     publicUrl: Optional[str] = None | ||||
|  | ||||
| @ -1506,6 +1506,7 @@ class CrawlOperator(BaseOperator): | ||||
|             await self.org_ops.inc_org_bytes_stored( | ||||
|                 crawl.oid, status.filesAddedSize, "crawl" | ||||
|             ) | ||||
|             await self.org_ops.set_last_crawl_finished(crawl.oid) | ||||
|             await self.coll_ops.add_successful_crawl_to_collections(crawl.id, crawl.cid) | ||||
| 
 | ||||
|         if state in FAILED_STATES: | ||||
|  | ||||
| @ -221,12 +221,25 @@ class OrgOps: | ||||
|         sort_query = {"default": -1} | ||||
| 
 | ||||
|         if sort_by: | ||||
|             sort_fields = ("name", "slug", "readOnly") | ||||
|             sort_fields = ( | ||||
|                 "name", | ||||
|                 "slug", | ||||
|                 "readOnly", | ||||
|                 "lastCrawlFinished", | ||||
|                 "subscriptionStatus", | ||||
|                 "subscriptionPlan", | ||||
|             ) | ||||
|             if sort_by not in sort_fields: | ||||
|                 raise HTTPException(status_code=400, detail="invalid_sort_by") | ||||
|             if sort_direction not in (1, -1): | ||||
|                 raise HTTPException(status_code=400, detail="invalid_sort_direction") | ||||
| 
 | ||||
|             if sort_by == "subscriptionStatus": | ||||
|                 sort_by = "subscription.status" | ||||
| 
 | ||||
|             if sort_by == "subscriptionPlan": | ||||
|                 sort_by = "subscription.planId" | ||||
| 
 | ||||
|             # Do lexical sort of names | ||||
|             if sort_by == "name": | ||||
|                 sort_by = "nameLower" | ||||
| @ -1382,6 +1395,14 @@ class OrgOps: | ||||
| 
 | ||||
|         return {"success": True} | ||||
| 
 | ||||
|     async def set_last_crawl_finished(self, oid: UUID): | ||||
|         """Recalculate and set lastCrawlFinished field on org""" | ||||
|         last_crawl_finished = await self.base_crawl_ops.get_org_last_crawl_finished(oid) | ||||
|         await self.orgs.find_one_and_update( | ||||
|             {"_id": oid}, | ||||
|             {"$set": {"lastCrawlFinished": last_crawl_finished}}, | ||||
|         ) | ||||
| 
 | ||||
| 
 | ||||
| # ============================================================================ | ||||
| # pylint: disable=too-many-statements, too-many-arguments | ||||
|  | ||||
| @ -755,3 +755,23 @@ def test_sort_orgs(admin_auth_headers): | ||||
|         if last_name: | ||||
|             assert org_name_lower < last_name | ||||
|         last_name = org_name_lower | ||||
| 
 | ||||
|     # Sort desc by lastCrawlFinished, ensure default org still first | ||||
|     r = requests.get( | ||||
|         f"{API_PREFIX}/orgs?sortBy=lastCrawlFinished&sortDirection=-1", | ||||
|         headers=admin_auth_headers, | ||||
|     ) | ||||
|     data = r.json() | ||||
|     orgs = data["items"] | ||||
| 
 | ||||
|     assert orgs[0]["default"] | ||||
| 
 | ||||
|     other_orgs = orgs[1:] | ||||
|     last_last_crawl_finished = None | ||||
|     for org in other_orgs: | ||||
|         last_crawl_finished = org.get("lastCrawlFinished") | ||||
|         if not last_crawl_finished: | ||||
|             continue | ||||
|         if last_last_crawl_finished: | ||||
|             assert last_crawl_finished <= last_last_crawl_finished | ||||
|         last_last_crawl_finished = last_crawl_finished | ||||
|  | ||||
| @ -14,6 +14,7 @@ import { when } from "lit/directives/when.js"; | ||||
| 
 | ||||
| import { BtrixElement } from "@/classes/BtrixElement"; | ||||
| import type { Dialog } from "@/components/ui/dialog"; | ||||
| import { SubscriptionStatus } from "@/types/billing"; | ||||
| import type { ProxiesAPIResponse, Proxy } from "@/types/crawler"; | ||||
| import type { OrgData } from "@/utils/orgs"; | ||||
| 
 | ||||
| @ -26,7 +27,7 @@ import type { OrgData } from "@/utils/orgs"; | ||||
| export class OrgsList extends BtrixElement { | ||||
|   static styles = css` | ||||
|     btrix-table { | ||||
|       grid-template-columns: min-content [clickable-start] 50ch auto auto auto [clickable-end] min-content; | ||||
|       grid-template-columns: min-content [clickable-start] minmax(auto, 50ch) auto auto auto [clickable-end] min-content; | ||||
|     } | ||||
|   `;
 | ||||
| 
 | ||||
| @ -130,6 +131,9 @@ export class OrgsList extends BtrixElement { | ||||
|           <btrix-table-header-cell class="px-2"> | ||||
|             ${msg("Bytes Stored")} | ||||
|           </btrix-table-header-cell> | ||||
|           <btrix-table-header-cell class="px-2"> | ||||
|             ${msg("Last Crawl")} | ||||
|           </btrix-table-header-cell> | ||||
|           <btrix-table-header-cell> | ||||
|             <span class="sr-only">${msg("Actions")}</span> | ||||
|           </btrix-table-header-cell> | ||||
| @ -644,16 +648,82 @@ export class OrgsList extends BtrixElement { | ||||
|       }; | ||||
|     } | ||||
| 
 | ||||
|     let subscription = { | ||||
|       icon: none, | ||||
|       description: msg("No Subscription"), | ||||
|     }; | ||||
| 
 | ||||
|     if (org.subscription) { | ||||
|       switch (org.subscription.status) { | ||||
|         case SubscriptionStatus.Active: | ||||
|           subscription = { | ||||
|             icon: html`<sl-icon
 | ||||
|               class="text-base text-success" | ||||
|               name="credit-card-fill" | ||||
|               label=${msg("Active Subscription")} | ||||
|             ></sl-icon>`, | ||||
|             description: msg("Active Subscription"), | ||||
|           }; | ||||
|           break; | ||||
|         case SubscriptionStatus.Trialing: | ||||
|           subscription = { | ||||
|             icon: html`<sl-icon
 | ||||
|               class="text-base text-neutral-400" | ||||
|               name="basket-fill" | ||||
|               label=${msg("Trial")} | ||||
|             ></sl-icon>`, | ||||
|             description: msg("Trial"), | ||||
|           }; | ||||
|           break; | ||||
|         case SubscriptionStatus.TrialingCanceled: | ||||
|           subscription = { | ||||
|             icon: html`<sl-icon
 | ||||
|               class="text-base text-neutral-400" | ||||
|               name="x-square-fill" | ||||
|               label=${msg("Trial Cancelled")} | ||||
|             ></sl-icon>`, | ||||
|             description: msg("Trial Canceled"), | ||||
|           }; | ||||
|           break; | ||||
|         case SubscriptionStatus.PausedPaymentFailed: | ||||
|           subscription = { | ||||
|             icon: html`<sl-icon
 | ||||
|               class="text-base text-danger" | ||||
|               name="exclamation-triangle-fill" | ||||
|               label=${msg("Payment Failed")} | ||||
|             ></sl-icon>`, | ||||
|             description: msg("Payment Failed"), | ||||
|           }; | ||||
|           break; | ||||
|         case SubscriptionStatus.Cancelled: | ||||
|           subscription = { | ||||
|             icon: html`<sl-icon
 | ||||
|               class="text-base text-neutral-400" | ||||
|               name="x-square-fill" | ||||
|               label=${msg("Canceled")} | ||||
|             > | ||||
|             </sl-icon>`, | ||||
|             description: msg("Canceled"), | ||||
|           }; | ||||
|           break; | ||||
|         default: | ||||
|           break; | ||||
|       } | ||||
|     } | ||||
| 
 | ||||
|     return html` | ||||
|       <btrix-table-row | ||||
|         class="${isUserOrg | ||||
|           ? "" | ||||
|           : "opacity-50"} cursor-pointer select-none border-b bg-neutral-0 transition-colors first-of-type:rounded-t last-of-type:rounded-b last-of-type:border-none focus-within:bg-neutral-50 hover:bg-neutral-50" | ||||
|       > | ||||
|         <btrix-table-cell class="min-w-6 pl-2"> | ||||
|         <btrix-table-cell class="min-w-6 gap-1 pl-2"> | ||||
|           <sl-tooltip content=${status.description}> | ||||
|             ${status.icon} | ||||
|           </sl-tooltip> | ||||
|           <sl-tooltip content=${subscription.description}> | ||||
|             ${subscription.icon} | ||||
|           </sl-tooltip> | ||||
|         </btrix-table-cell> | ||||
|         <btrix-table-cell class="p-2" rowClickTarget="a"> | ||||
|           <a | ||||
| @ -670,7 +740,6 @@ export class OrgsList extends BtrixElement { | ||||
|               : org.name} | ||||
|           </a> | ||||
|         </btrix-table-cell> | ||||
| 
 | ||||
|         <btrix-table-cell class="p-2"> | ||||
|           ${this.localize.date(org.created, { dateStyle: "short" })} | ||||
|         </btrix-table-cell> | ||||
| @ -682,6 +751,11 @@ export class OrgsList extends BtrixElement { | ||||
|             ? this.localize.bytes(org.bytesStored, { unitDisplay: "narrow" }) | ||||
|             : none} | ||||
|         </btrix-table-cell> | ||||
|         <btrix-table-cell class="p-2"> | ||||
|           ${org.lastCrawlFinished | ||||
|             ? this.localize.date(org.lastCrawlFinished, { dateStyle: "short" }) | ||||
|             : none} | ||||
|         </btrix-table-cell> | ||||
|         <btrix-table-cell class="p-1"> | ||||
|           <btrix-overflow-dropdown | ||||
|             @click=${(e: MouseEvent) => e.stopPropagation()} | ||||
|  | ||||
| @ -242,7 +242,7 @@ export class ShareCollection extends BtrixElement { | ||||
|         @sl-after-hide=${() => { | ||||
|           this.tabGroup?.show(Tab.Link); | ||||
|         }} | ||||
|         class="[--body-spacing:0] [--width:40rem]" | ||||
|         class="[--width:40rem] [--body-spacing:0]" | ||||
|       > | ||||
|         <sl-tab-group> | ||||
|           <sl-tab slot="nav" panel=${Tab.Link} | ||||
|  | ||||
| @ -94,6 +94,7 @@ export const orgDataSchema = z.object({ | ||||
|   crawlingDefaults: crawlingDefaultsSchema.nullable(), | ||||
|   allowSharedProxies: z.boolean(), | ||||
|   allowedProxies: z.array(z.string()), | ||||
|   lastCrawlFinished: apiDateSchema.nullable(), | ||||
|   enablePublicProfile: z.boolean(), | ||||
|   publicDescription: z.string().nullable(), | ||||
|   publicUrl: z.string().nullable(), | ||||
|  | ||||
		Loading…
	
		Reference in New Issue
	
	Block a user