Add last crawl and subscription status indicators to org list (#2273)

Fixes #2260 

- Adds `lastCrawlFinished` to Organization model, updated after crawls
are added/deleted and with an idempotent migration to backfill existing
orgs
- Adds Last Crawl column to end of admin orgs list table
- Adds subscription icon next to existing status icon in orgs list
- Adds "lastCrawlFinished", "subscriptionStatus", and "subscriptionPlan"
sort options to orgs list backend endpoint in anticipation of future
sorting/filtering of orgs list

---------

Co-authored-by: emma <hi@emma.cafe>
Co-authored-by: Henry Wilkinson <henry@wilkinson.graphics>
Co-authored-by: Ilya Kreymer <ikreymer@gmail.com>

backend/btrixcloud/basecrawls.py

@@ -1,6 +1,6 @@
 """ base crawl type """
 
-from datetime import timedelta
+from datetime import datetime, timedelta
 from typing import Optional, List, Union, Dict, Any, Type, TYPE_CHECKING, cast, Tuple
 from uuid import UUID
 import urllib.parse
@@ -359,6 +359,8 @@ class BaseCrawlOps:
 
         await self.orgs.inc_org_bytes_stored(org.id, -size, type_)
 
+        await self.orgs.set_last_crawl_finished(org.id)
+
         quota_reached = self.orgs.storage_quota_reached(org)
 
         return res.deleted_count, cids_to_update, quota_reached
@@ -853,6 +855,21 @@ class BaseCrawlOps:
 
         return total_size, crawls_size, uploads_size
 
+    async def get_org_last_crawl_finished(self, oid: UUID) -> Optional[datetime]:
+        """Get last crawl finished time for org"""
+        last_crawl_finished: Optional[datetime] = None
+
+        cursor = (
+            self.crawls.find({"oid": oid, "finished": {"$ne": None}})
+            .sort({"finished": -1})
+            .limit(1)
+        )
+        last_crawl = await cursor.to_list(length=1)
+        if last_crawl:
+            last_crawl_finished = last_crawl[0].get("finished")
+
+        return last_crawl_finished
+
 
 # ============================================================================
 def init_base_crawls_api(app, user_dep, *args):

backend/btrixcloud/db.py

@@ -17,7 +17,7 @@ from pymongo.errors import InvalidName
 from .migrations import BaseMigration
 
 
-CURR_DB_VERSION = "0037"
+CURR_DB_VERSION = "0038"
 
 
 # ============================================================================
@@ -95,8 +95,11 @@ async def update_and_prepare_db(
     """
     await ping_db(mdb)
     print("Database setup started", flush=True)
-    if await run_db_migrations(mdb, user_manager, background_job_ops, page_ops):
+    if await run_db_migrations(
+        mdb, user_manager, page_ops, org_ops, background_job_ops
+    ):
         await drop_indexes(mdb)
+
     await create_indexes(
         org_ops,
         crawl_ops,
@@ -114,7 +117,8 @@ async def update_and_prepare_db(
 
 
 # ============================================================================
-async def run_db_migrations(mdb, user_manager, background_job_ops, page_ops):
+# pylint: disable=too-many-locals
+async def run_db_migrations(mdb, user_manager, page_ops, org_ops, background_job_ops):
     """Run database migrations."""
 
     # if first run, just set version and exit
@@ -147,7 +151,10 @@ async def run_db_migrations(mdb, user_manager, background_job_ops, page_ops):
             migration_module = importlib.util.module_from_spec(spec)
             spec.loader.exec_module(migration_module)
             migration = migration_module.Migration(
-                mdb, background_job_ops=background_job_ops, page_ops=page_ops
+                mdb,
+                page_ops=page_ops,
+                org_ops=org_ops,
+                background_job_ops=background_job_ops,
             )
             if await migration.run():
                 migrations_run = True

backend/btrixcloud/migrations/migration_0038_org_last_crawl_finished.py

@@ -0,0 +1,43 @@
+"""
+Migration 0038 - Organization lastCrawlFinished field
+"""
+
+from btrixcloud.migrations import BaseMigration
+
+
+MIGRATION_VERSION = "0038"
+
+
+class Migration(BaseMigration):
+    """Migration class."""
+
+    # pylint: disable=unused-argument
+    def __init__(self, mdb, **kwargs):
+        super().__init__(mdb, migration_version=MIGRATION_VERSION)
+
+        self.org_ops = kwargs.get("org_ops")
+
+    async def migrate_up(self):
+        """Perform migration up. Set lastCrawlFinished for each org."""
+        # pylint: disable=duplicate-code, line-too-long
+        if self.org_ops is None:
+            print(
+                "Unable to set lastCrawlFinished for orgs, missing org_ops", flush=True
+            )
+            return
+
+        orgs_db = self.mdb["organizations"]
+        async for org_dict in orgs_db.find({}):
+            oid = org_dict.get("_id")
+
+            if org_dict.get("lastCrawlFinished"):
+                continue
+
+            try:
+                await self.org_ops.set_last_crawl_finished(oid)
+            # pylint: disable=broad-exception-caught
+            except Exception as err:
+                print(
+                    f"Error setting lastCrawlFinished for org {oid}: {err}",
+                    flush=True,
+                )

backend/btrixcloud/models.py

@@ -1723,6 +1723,8 @@ class OrgOut(BaseMongoModel):
     allowedProxies: list[str] = []
     crawlingDefaults: Optional[CrawlConfigDefaults] = None
 
+    lastCrawlFinished: Optional[datetime] = None
+
     enablePublicProfile: bool = False
     publicDescription: str = ""
     publicUrl: str = ""
@@ -1782,6 +1784,8 @@ class Organization(BaseMongoModel):
     allowedProxies: list[str] = []
     crawlingDefaults: Optional[CrawlConfigDefaults] = None
 
+    lastCrawlFinished: Optional[datetime] = None
+
     enablePublicProfile: bool = False
     publicDescription: Optional[str] = None
     publicUrl: Optional[str] = None

backend/btrixcloud/operator/crawls.py

@@ -1506,6 +1506,7 @@ class CrawlOperator(BaseOperator):
             await self.org_ops.inc_org_bytes_stored(
                 crawl.oid, status.filesAddedSize, "crawl"
             )
+            await self.org_ops.set_last_crawl_finished(crawl.oid)
             await self.coll_ops.add_successful_crawl_to_collections(crawl.id, crawl.cid)
 
         if state in FAILED_STATES:

backend/btrixcloud/orgs.py

@@ -221,12 +221,25 @@ class OrgOps:
         sort_query = {"default": -1}
 
         if sort_by:
-            sort_fields = ("name", "slug", "readOnly")
+            sort_fields = (
+                "name",
+                "slug",
+                "readOnly",
+                "lastCrawlFinished",
+                "subscriptionStatus",
+                "subscriptionPlan",
+            )
             if sort_by not in sort_fields:
                 raise HTTPException(status_code=400, detail="invalid_sort_by")
             if sort_direction not in (1, -1):
                 raise HTTPException(status_code=400, detail="invalid_sort_direction")
 
+            if sort_by == "subscriptionStatus":
+                sort_by = "subscription.status"
+
+            if sort_by == "subscriptionPlan":
+                sort_by = "subscription.planId"
+
             # Do lexical sort of names
             if sort_by == "name":
                 sort_by = "nameLower"
@@ -1382,6 +1395,14 @@ class OrgOps:
 
         return {"success": True}
 
+    async def set_last_crawl_finished(self, oid: UUID):
+        """Recalculate and set lastCrawlFinished field on org"""
+        last_crawl_finished = await self.base_crawl_ops.get_org_last_crawl_finished(oid)
+        await self.orgs.find_one_and_update(
+            {"_id": oid},
+            {"$set": {"lastCrawlFinished": last_crawl_finished}},
+        )
+
 
 # ============================================================================
 # pylint: disable=too-many-statements, too-many-arguments

backend/test/test_org.py

@@ -755,3 +755,23 @@ def test_sort_orgs(admin_auth_headers):
         if last_name:
             assert org_name_lower < last_name
         last_name = org_name_lower
+
+    # Sort desc by lastCrawlFinished, ensure default org still first
+    r = requests.get(
+        f"{API_PREFIX}/orgs?sortBy=lastCrawlFinished&sortDirection=-1",
+        headers=admin_auth_headers,
+    )
+    data = r.json()
+    orgs = data["items"]
+
+    assert orgs[0]["default"]
+
+    other_orgs = orgs[1:]
+    last_last_crawl_finished = None
+    for org in other_orgs:
+        last_crawl_finished = org.get("lastCrawlFinished")
+        if not last_crawl_finished:
+            continue
+        if last_last_crawl_finished:
+            assert last_crawl_finished <= last_last_crawl_finished
+        last_last_crawl_finished = last_crawl_finished

frontend/src/components/orgs-list.ts

@@ -14,6 +14,7 @@ import { when } from "lit/directives/when.js";
 
 import { BtrixElement } from "@/classes/BtrixElement";
 import type { Dialog } from "@/components/ui/dialog";
+import { SubscriptionStatus } from "@/types/billing";
 import type { ProxiesAPIResponse, Proxy } from "@/types/crawler";
 import type { OrgData } from "@/utils/orgs";
 
@@ -26,7 +27,7 @@ import type { OrgData } from "@/utils/orgs";
 export class OrgsList extends BtrixElement {
   static styles = css`
     btrix-table {
-      grid-template-columns: min-content [clickable-start] 50ch auto auto auto [clickable-end] min-content;
+      grid-template-columns: min-content [clickable-start] minmax(auto, 50ch) auto auto auto [clickable-end] min-content;
     }
   `;
 
@@ -130,6 +131,9 @@ export class OrgsList extends BtrixElement {
           <btrix-table-header-cell class="px-2">
             ${msg("Bytes Stored")}
           </btrix-table-header-cell>
+          <btrix-table-header-cell class="px-2">
+            ${msg("Last Crawl")}
+          </btrix-table-header-cell>
           <btrix-table-header-cell>
             <span class="sr-only">${msg("Actions")}</span>
           </btrix-table-header-cell>
@@ -644,16 +648,82 @@ export class OrgsList extends BtrixElement {
       };
     }
 
+    let subscription = {
+      icon: none,
+      description: msg("No Subscription"),
+    };
+
+    if (org.subscription) {
+      switch (org.subscription.status) {
+        case SubscriptionStatus.Active:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-success"
+              name="credit-card-fill"
+              label=${msg("Active Subscription")}
+            ></sl-icon>`,
+            description: msg("Active Subscription"),
+          };
+          break;
+        case SubscriptionStatus.Trialing:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-neutral-400"
+              name="basket-fill"
+              label=${msg("Trial")}
+            ></sl-icon>`,
+            description: msg("Trial"),
+          };
+          break;
+        case SubscriptionStatus.TrialingCanceled:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-neutral-400"
+              name="x-square-fill"
+              label=${msg("Trial Cancelled")}
+            ></sl-icon>`,
+            description: msg("Trial Canceled"),
+          };
+          break;
+        case SubscriptionStatus.PausedPaymentFailed:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-danger"
+              name="exclamation-triangle-fill"
+              label=${msg("Payment Failed")}
+            ></sl-icon>`,
+            description: msg("Payment Failed"),
+          };
+          break;
+        case SubscriptionStatus.Cancelled:
+          subscription = {
+            icon: html`<sl-icon
+              class="text-base text-neutral-400"
+              name="x-square-fill"
+              label=${msg("Canceled")}
+            >
+            </sl-icon>`,
+            description: msg("Canceled"),
+          };
+          break;
+        default:
+          break;
+      }
+    }
+
     return html`
       <btrix-table-row
         class="${isUserOrg
           ? ""
           : "opacity-50"} cursor-pointer select-none border-b bg-neutral-0 transition-colors first-of-type:rounded-t last-of-type:rounded-b last-of-type:border-none focus-within:bg-neutral-50 hover:bg-neutral-50"
       >
-        <btrix-table-cell class="min-w-6 pl-2">
+        <btrix-table-cell class="min-w-6 gap-1 pl-2">
           <sl-tooltip content=${status.description}>
             ${status.icon}
           </sl-tooltip>
+          <sl-tooltip content=${subscription.description}>
+            ${subscription.icon}
+          </sl-tooltip>
         </btrix-table-cell>
         <btrix-table-cell class="p-2" rowClickTarget="a">
           <a
@@ -670,7 +740,6 @@ export class OrgsList extends BtrixElement {
               : org.name}
           </a>
         </btrix-table-cell>
-
         <btrix-table-cell class="p-2">
           ${this.localize.date(org.created, { dateStyle: "short" })}
         </btrix-table-cell>
@@ -682,6 +751,11 @@ export class OrgsList extends BtrixElement {
             ? this.localize.bytes(org.bytesStored, { unitDisplay: "narrow" })
             : none}
         </btrix-table-cell>
+        <btrix-table-cell class="p-2">
+          ${org.lastCrawlFinished
+            ? this.localize.date(org.lastCrawlFinished, { dateStyle: "short" })
+            : none}
+        </btrix-table-cell>
         <btrix-table-cell class="p-1">
           <btrix-overflow-dropdown
             @click=${(e: MouseEvent) => e.stopPropagation()}

frontend/src/features/collections/share-collection.ts

@@ -242,7 +242,7 @@ export class ShareCollection extends BtrixElement {
         @sl-after-hide=${() => {
           this.tabGroup?.show(Tab.Link);
         }}
-        class="[--body-spacing:0] [--width:40rem]"
+        class="[--width:40rem] [--body-spacing:0]"
       >
         <sl-tab-group>
           <sl-tab slot="nav" panel=${Tab.Link}

frontend/src/types/org.ts

@@ -94,6 +94,7 @@ export const orgDataSchema = z.object({
   crawlingDefaults: crawlingDefaultsSchema.nullable(),
   allowSharedProxies: z.boolean(),
   allowedProxies: z.array(z.string()),
+  lastCrawlFinished: apiDateSchema.nullable(),
   enablePublicProfile: z.boolean(),
   publicDescription: z.string().nullable(),
   publicUrl: z.string().nullable(),