Issue all non-upload storage-quota-update events from LiteElement (#1151)

- More specific toast notification error messages to the action being attempted
- Single dismissable global banner shown when org storage is reached
- Removed check for storage quota reached in `runNow`, since buttons are disabled in UI, and errors handled if request fails.
- Allow creating new workflow when storage quota reached
- More responsive storage quota updates: add storageQuotaReached to archived item replay.json, updates w/o reload when crawl pushes quota over limit
- Modify LiteElement to check for storageQuotaReached on GET requests

---------
Co-authored-by: sua yoo <sua@suayoo.com>

backend/btrixcloud/basecrawls.py

@@ -24,7 +24,7 @@ from .models import (
     PaginatedResponse,
     User,
 )
-from .orgs import inc_org_bytes_stored
+from .orgs import inc_org_bytes_stored, storage_quota_reached
 from .pagination import paginated_format, DEFAULT_PAGE_SIZE
 from .storages import get_presigned_url, delete_crawl_file_object
 from .utils import dt_now, get_redis_crawl_stats
@@ -127,6 +127,8 @@ class BaseCrawlOps:
             # pylint: disable=invalid-name
             crawl.userName = user.name
 
+        crawl.storageQuotaReached = await storage_quota_reached(self.orgs_db, crawl.oid)
+
         return crawl
 
     async def get_resource_resolved_raw_crawl(

backend/btrixcloud/models.py

@@ -372,6 +372,8 @@ class CrawlOut(BaseMongoModel):
     manual: Optional[bool]
     cid_rev: Optional[int]
 
+    storageQuotaReached: Optional[bool]
+
 
 # ============================================================================
 class CrawlOutWithResources(CrawlOut):

frontend/src/components/file-uploader.ts

@@ -459,13 +459,13 @@ export class FileUploader extends LiteElement {
       }
     } catch (err: any) {
       if (err === ABORT_REASON_USER_CANCEL) {
-        console.debug("Fetch crawls aborted to user cancel");
+        console.debug("Upload aborted to user cancel");
       } else {
         let message = msg("Sorry, couldn't upload file at this time.");
         console.debug(err);
         if (err === ABORT_REASON_QUOTA_REACHED) {
           message = msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
+            "Your org does not have enough storage to upload this file."
           );
           this.dispatchEvent(
             new CustomEvent("storage-quota-update", {
@@ -474,7 +474,6 @@ export class FileUploader extends LiteElement {
             })
           );
         }
-        console.debug(err);
         this.notify({
           message: message,
           variant: "danger",

frontend/src/pages/org/browser-profiles-detail.ts

@@ -637,18 +637,13 @@ export class BrowserProfilesDetail extends LiteElement {
       if (e.isApiError && e.statusCode === 403) {
         if (e.details === "storage_quota_reached") {
           message = msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
+            "Your org does not have enough storage to save this browser profile."
-          );
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: true },
-              bubbles: true,
-            })
           );
         } else {
           message = msg("You do not have permission to edit browser profiles.");
         }
       }
+
       this.notify({
         message: message,
         variant: "danger",

frontend/src/pages/org/browser-profiles-new.ts

@@ -204,18 +204,13 @@ export class BrowserProfilesNew extends LiteElement {
       this.navTo(`/orgs/${this.orgId}/browser-profiles/profile/${data.id}`);
     } catch (e: any) {
       this.isSubmitting = false;
+
       let message = msg("Sorry, couldn't create browser profile at this time.");
 
       if (e.isApiError && e.statusCode === 403) {
         if (e.details === "storage_quota_reached") {
           message = msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
+            "Your org does not have enough storage to save this browser profile."
-          );
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: true },
-              bubbles: true,
-            })
           );
         } else {
           message = msg(
@@ -223,6 +218,7 @@ export class BrowserProfilesNew extends LiteElement {
           );
         }
       }
+
       this.notify({
         message: message,
         variant: "danger",

frontend/src/pages/org/index.ts

@@ -79,6 +79,9 @@ export class Org extends LiteElement {
   @state()
   private orgStorageQuotaReached = false;
 
+  @state()
+  private showStorageQuotaAlert = false;
+
   @state()
   private org?: OrgData | null;
 
@@ -162,7 +165,7 @@ export class Org extends LiteElement {
     }
 
     return html`
-      ${this.renderOrgNavBar()}
+      ${this.renderStorageAlert()} ${this.renderOrgNavBar()}
       <main>
         <div
           class="w-full max-w-screen-lg mx-auto px-3 box-border py-5"
@@ -174,6 +177,32 @@ export class Org extends LiteElement {
     `;
   }
 
+  private renderStorageAlert() {
+    return html`
+      <div
+        class="transition-all ${this.showStorageQuotaAlert
+          ? "bg-slate-100 border-b py-5"
+          : ""}"
+      >
+        <div class="w-full max-w-screen-lg mx-auto px-3 box-border">
+          <sl-alert
+            variant="warning"
+            closable
+            ?open=${this.showStorageQuotaAlert}
+            @sl-after-hide=${() => (this.showStorageQuotaAlert = false)}
+          >
+            <sl-icon slot="icon" name="exclamation-triangle"></sl-icon>
+            <strong>${msg("Your org has reached its storage limit")}</strong
+            ><br />
+            ${msg(
+              "To run crawls again, delete unneeded archived items and unused browser profiles to free up space, or contact us to upgrade your storage plan."
+            )}
+          </sl-alert>
+        </div>
+      </div>
+    `;
+  }
+
   private renderOrgNavBar() {
     return html`
       <div class="w-full max-w-screen-lg mx-auto px-3 box-border">
@@ -439,8 +468,12 @@ export class Org extends LiteElement {
   }
 
   private async onStorageQuotaUpdate(e: CustomEvent) {
+    e.stopPropagation();
     const { reached } = e.detail;
     this.orgStorageQuotaReached = reached;
+    if (reached) {
+      this.showStorageQuotaAlert = true;
+    }
   }
 
   private async onUserRoleChange(e: UserRoleChangeEvent) {
@@ -549,5 +582,9 @@ export class Org extends LiteElement {
     } else {
       this.orgStorageQuotaReached = false;
     }
+
+    if (this.orgStorageQuotaReached) {
+      this.showStorageQuotaAlert = true;
+    }
   }
 }

frontend/src/pages/org/workflow-detail.ts

@@ -633,6 +633,7 @@ export class WorkflowDetail extends LiteElement {
             () => html`
               <sl-menu-item
                 style="--sl-color-neutral-700: var(--success)"
+                ?disabled=${this.orgStorageQuotaReached}
                 @click=${() => this.runNow()}
               >
                 <sl-icon name="play" slot="prefix"></sl-icon>
@@ -1430,17 +1431,6 @@ export class WorkflowDetail extends LiteElement {
   }
 
   private async runNow(): Promise<void> {
-    if (this.orgStorageQuotaReached) {
-      this.notify({
-        message: msg(
-          "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-        ),
-        variant: "danger",
-        icon: "exclamation-octagon",
-      });
-      return;
-    }
-
     try {
       const data = await this.apiFetch(
         `/orgs/${this.orgId}/crawlconfigs/${this.workflow!.id}/run`,
@@ -1465,15 +1455,7 @@ export class WorkflowDetail extends LiteElement {
       let message = msg("Sorry, couldn't run crawl at this time.");
       if (e.isApiError && e.statusCode === 403) {
         if (e.details === "storage_quota_reached") {
-          message = msg(
+          message = msg("Your org does not have enough storage to run crawls.");
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-          );
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: true },
-              bubbles: true,
-            })
-          );
         } else {
           message = msg("You do not have permission to run crawls.");
         }

frontend/src/pages/org/workflow-editor.ts

@@ -2037,27 +2037,28 @@ https://archiveweb.page/images/${"logo.svg"}`}
       const crawlId = data.run_now_job;
       const storageQuotaReached = data.storageQuotaReached;
 
-      let message = msg("Workflow created.");
+      if (crawlId && storageQuotaReached) {
-      if (crawlId && !storageQuotaReached) {
-        message = msg("Crawl started with new template.");
-      } else if (this.configId) {
-        message = msg("Workflow updated.");
-      }
-
-      this.notify({
-        message,
-        variant: "success",
-        icon: "check2-circle",
-        duration: 8000,
-      });
-
-      if (storageQuotaReached) {
         this.notify({
+          title: msg("Workflow saved."),
           message: msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
+            "Could not start crawl with new workflow settings due to storage quota."
           ),
-          variant: "danger",
+          variant: "warning",
-          icon: "exclamation-octagon",
+          icon: "exclamation-triangle",
+          duration: 8000,
+        });
+      } else {
+        let message = msg("Workflow created.");
+        if (crawlId) {
+          message = msg("Crawl started with new workflow settings.");
+        } else if (this.configId) {
+          message = msg("Workflow updated.");
+        }
+
+        this.notify({
+          message,
+          variant: "success",
+          icon: "check2-circle",
         });
       }
 

frontend/src/pages/org/workflows-list.ts

@@ -199,21 +199,15 @@ export class WorkflowsList extends LiteElement {
           ${when(
             this.isCrawler,
             () => html`
-              <sl-tooltip
+              <sl-button
-                content=${msg("Org Storage Full")}
+                href=${`/orgs/${this.orgId}/workflows?new&jobType=`}
-                ?disabled=${!this.orgStorageQuotaReached}
+                variant="primary"
+                size="small"
+                @click=${this.navLink}
               >
-                <sl-button
+                <sl-icon slot="prefix" name="plus-lg"></sl-icon>
-                  href=${`/orgs/${this.orgId}/workflows?new&jobType=`}
+                ${msg("New Workflow")}
-                  variant="primary"
+              </sl-button>
-                  size="small"
-                  ?disabled=${this.orgStorageQuotaReached}
-                  @click=${this.navLink}
-                >
-                  <sl-icon slot="prefix" name="plus-lg"></sl-icon>
-                  ${msg("New Workflow")}
-                </sl-button>
-              </sl-tooltip>
             `
           )}
         </div>
@@ -434,6 +428,7 @@ export class WorkflowsList extends LiteElement {
         () => html`
           <sl-menu-item
             style="--sl-color-neutral-700: var(--success)"
+            ?disabled=${this.orgStorageQuotaReached}
             @click=${() => this.runNow(workflow)}
           >
             <sl-icon name="play" slot="prefix"></sl-icon>
@@ -745,17 +740,6 @@ export class WorkflowsList extends LiteElement {
   }
 
   private async runNow(workflow: Workflow): Promise<void> {
-    if (this.orgStorageQuotaReached) {
-      this.notify({
-        message: msg(
-          "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-        ),
-        variant: "danger",
-        icon: "exclamation-octagon",
-      });
-      return;
-    }
-
     try {
       const data = await this.apiFetch(
         `/orgs/${this.orgId}/crawlconfigs/${workflow.id}/run`,
@@ -788,15 +772,7 @@ export class WorkflowsList extends LiteElement {
       let message = msg("Sorry, couldn't run crawl at this time.");
       if (e.isApiError && e.statusCode === 403) {
         if (e.details === "storage_quota_reached") {
-          message = msg(
+          message = msg("Your org does not have enough storage to run crawls.");
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-          );
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: true },
-              bubbles: true,
-            })
-          );
         } else {
           message = msg("You do not have permission to run crawls.");
         }

frontend/src/utils/LiteElement.ts

@@ -123,21 +123,55 @@ export default class LiteElement extends LitElement {
       ...opts,
     });
 
-    if (resp.status !== 200) {
+    if (resp.ok) {
-      if (resp.status === 401) {
+      const body = await resp.json();
-        this.dispatchEvent(new CustomEvent("need-login"));
+      const storageQuotaReached = body.storageQuotaReached;
+      if (typeof storageQuotaReached === "boolean") {
+        this.dispatchEvent(
+          new CustomEvent("storage-quota-update", {
+            detail: { reached: storageQuotaReached },
+            bubbles: true,
+          })
+        );
       }
 
-      let detail;
+      return body;
-      let errorMessage: string = msg("Unknown API error");
+    }
 
-      try {
+    let errorDetail;
-        detail = (await resp.json()).detail;
+    try {
+      errorDetail = (await resp.json()).detail;
+    } catch {}
 
-        if (typeof detail === "string") {
+    let errorMessage: string = msg("Unknown API error");
-          errorMessage = detail;
+
-        } else if (Array.isArray(detail) && detail.length) {
+    switch (resp.status) {
-          const fieldDetail = detail[0];
+      case 401: {
+        this.dispatchEvent(new CustomEvent("need-login"));
+        errorMessage = msg("Need login");
+        break;
+      }
+      case 403: {
+        if (errorDetail === "storage_quota_reached") {
+          this.dispatchEvent(
+            new CustomEvent("storage-quota-update", {
+              detail: { reached: true },
+              bubbles: true,
+            })
+          );
+          errorMessage = msg("Storage quota reached");
+          break;
+        }
+      }
+      case 404: {
+        errorMessage = msg("Not found");
+        break;
+      }
+      default: {
+        if (typeof errorDetail === "string") {
+          errorMessage = errorDetail;
+        } else if (Array.isArray(errorDetail) && errorDetail.length) {
+          const fieldDetail = errorDetail[0] || {};
           const { loc, msg } = fieldDetail;
 
           const fieldName = loc
@@ -145,31 +179,14 @@ export default class LiteElement extends LitElement {
             .join(" ");
           errorMessage = `${fieldName} ${msg}`;
         }
-      } catch {}
+        break;
-
+      }
-      throw new APIError({
-        message: errorMessage,
-        status: resp.status,
-        details: detail,
-      });
     }
 
-    const body = await resp.json();
+    throw new APIError({
-
+      message: errorMessage,
-    if (options?.method && options?.method !== "GET" && resp.status === 200) {
+      status: resp.status,
-      try {
+      details: errorDetail,
-        const storageQuotaReached = body.storageQuotaReached;
+    });
-        if (typeof storageQuotaReached === "boolean") {
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: storageQuotaReached },
-              bubbles: true,
-            })
-          );
-        }
-      } catch {}
-    }
-
-    return await body;
   }
 }