Issue all non-upload storage-quota-update events from LiteElement (#1151)

- More specific toast notification error messages to the action being attempted - Single dismissable global banner shown when org storage is reached - Removed check for storage quota reached in `runNow`, since buttons are disabled in UI, and errors handled if request fails. - Allow creating new workflow when storage quota reached - More responsive storage quota updates: add storageQuotaReached to archived item replay.json, updates w/o reload when crawl pushes quota over limit - Modify LiteElement to check for storageQuotaReached on GET requests --------- Co-authored-by: sua yoo <sua@suayoo.com>
2023-09-11 21:17:48 -04:00 · 2023-09-11 21:17:48 -04:00 · 9377a6f456
commit 9377a6f456
parent ad9bca2e92
10 changed files with 133 additions and 126 deletions
--- a/backend/btrixcloud/basecrawls.py
+++ b/backend/btrixcloud/basecrawls.py
@ -24,7 +24,7 @@ from .models import (
    PaginatedResponse,
    User,
 )
-from .orgs import inc_org_bytes_stored
+from .orgs import inc_org_bytes_stored, storage_quota_reached
 from .pagination import paginated_format, DEFAULT_PAGE_SIZE
 from .storages import get_presigned_url, delete_crawl_file_object
 from .utils import dt_now, get_redis_crawl_stats
@ -127,6 +127,8 @@ class BaseCrawlOps:
            # pylint: disable=invalid-name
            crawl.userName = user.name

+        crawl.storageQuotaReached = await storage_quota_reached(self.orgs_db, crawl.oid)
+
        return crawl

    async def get_resource_resolved_raw_crawl(
--- a/backend/btrixcloud/models.py
+++ b/backend/btrixcloud/models.py
@ -372,6 +372,8 @@ class CrawlOut(BaseMongoModel):
    manual: Optional[bool]
    cid_rev: Optional[int]

+    storageQuotaReached: Optional[bool]
+

 # ============================================================================
 class CrawlOutWithResources(CrawlOut):
--- a/frontend/src/components/file-uploader.ts
+++ b/frontend/src/components/file-uploader.ts
@ -459,13 +459,13 @@ export class FileUploader extends LiteElement {
      }
    } catch (err: any) {
      if (err === ABORT_REASON_USER_CANCEL) {
-        console.debug("Fetch crawls aborted to user cancel");
+        console.debug("Upload aborted to user cancel");
      } else {
        let message = msg("Sorry, couldn't upload file at this time.");
        console.debug(err);
        if (err === ABORT_REASON_QUOTA_REACHED) {
          message = msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
+            "Your org does not have enough storage to upload this file."
          );
          this.dispatchEvent(
            new CustomEvent("storage-quota-update", {
@ -474,7 +474,6 @@ export class FileUploader extends LiteElement {
            })
          );
        }
-        console.debug(err);
        this.notify({
          message: message,
          variant: "danger",
--- a/frontend/src/pages/org/browser-profiles-detail.ts
+++ b/frontend/src/pages/org/browser-profiles-detail.ts
@ -637,18 +637,13 @@ export class BrowserProfilesDetail extends LiteElement {
      if (e.isApiError && e.statusCode === 403) {
        if (e.details === "storage_quota_reached") {
          message = msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-          );
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: true },
-              bubbles: true,
-            })
+            "Your org does not have enough storage to save this browser profile."
          );
        } else {
          message = msg("You do not have permission to edit browser profiles.");
        }
      }
+
      this.notify({
        message: message,
        variant: "danger",
--- a/frontend/src/pages/org/browser-profiles-new.ts
+++ b/frontend/src/pages/org/browser-profiles-new.ts
@ -204,18 +204,13 @@ export class BrowserProfilesNew extends LiteElement {
      this.navTo(`/orgs/${this.orgId}/browser-profiles/profile/${data.id}`);
    } catch (e: any) {
      this.isSubmitting = false;
+
      let message = msg("Sorry, couldn't create browser profile at this time.");

      if (e.isApiError && e.statusCode === 403) {
        if (e.details === "storage_quota_reached") {
          message = msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-          );
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: true },
-              bubbles: true,
-            })
+            "Your org does not have enough storage to save this browser profile."
          );
        } else {
          message = msg(
@ -223,6 +218,7 @@ export class BrowserProfilesNew extends LiteElement {
          );
        }
      }
+
      this.notify({
        message: message,
        variant: "danger",
--- a/frontend/src/pages/org/index.ts
+++ b/frontend/src/pages/org/index.ts
@ -79,6 +79,9 @@ export class Org extends LiteElement {
  @state()
  private orgStorageQuotaReached = false;

+  @state()
+  private showStorageQuotaAlert = false;
+
  @state()
  private org?: OrgData | null;

@ -162,7 +165,7 @@ export class Org extends LiteElement {
    }

    return html`
-      ${this.renderOrgNavBar()}
+      ${this.renderStorageAlert()} ${this.renderOrgNavBar()}
      <main>
        <div
          class="w-full max-w-screen-lg mx-auto px-3 box-border py-5"
@ -174,6 +177,32 @@ export class Org extends LiteElement {
    `;
  }

+  private renderStorageAlert() {
+    return html`
+      <div
+        class="transition-all ${this.showStorageQuotaAlert
+          ? "bg-slate-100 border-b py-5"
+          : ""}"
+      >
+        <div class="w-full max-w-screen-lg mx-auto px-3 box-border">
+          <sl-alert
+            variant="warning"
+            closable
+            ?open=${this.showStorageQuotaAlert}
+            @sl-after-hide=${() => (this.showStorageQuotaAlert = false)}
+          >
+            <sl-icon slot="icon" name="exclamation-triangle"></sl-icon>
+            <strong>${msg("Your org has reached its storage limit")}</strong
+            ><br />
+            ${msg(
+              "To run crawls again, delete unneeded archived items and unused browser profiles to free up space, or contact us to upgrade your storage plan."
+            )}
+          </sl-alert>
+        </div>
+      </div>
+    `;
+  }
+
  private renderOrgNavBar() {
    return html`
      <div class="w-full max-w-screen-lg mx-auto px-3 box-border">
@ -439,8 +468,12 @@ export class Org extends LiteElement {
  }

  private async onStorageQuotaUpdate(e: CustomEvent) {
+    e.stopPropagation();
    const { reached } = e.detail;
    this.orgStorageQuotaReached = reached;
+    if (reached) {
+      this.showStorageQuotaAlert = true;
+    }
  }

  private async onUserRoleChange(e: UserRoleChangeEvent) {
@ -549,5 +582,9 @@ export class Org extends LiteElement {
    } else {
      this.orgStorageQuotaReached = false;
    }
+
+    if (this.orgStorageQuotaReached) {
+      this.showStorageQuotaAlert = true;
+    }
  }
 }
--- a/frontend/src/pages/org/workflow-detail.ts
+++ b/frontend/src/pages/org/workflow-detail.ts
@ -633,6 +633,7 @@ export class WorkflowDetail extends LiteElement {
            () => html`
              <sl-menu-item
                style="--sl-color-neutral-700: var(--success)"
+                ?disabled=${this.orgStorageQuotaReached}
                @click=${() => this.runNow()}
              >
                <sl-icon name="play" slot="prefix"></sl-icon>
@ -1430,17 +1431,6 @@ export class WorkflowDetail extends LiteElement {
  }

  private async runNow(): Promise<void> {
-    if (this.orgStorageQuotaReached) {
-      this.notify({
-        message: msg(
-          "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-        ),
-        variant: "danger",
-        icon: "exclamation-octagon",
-      });
-      return;
-    }
-
    try {
      const data = await this.apiFetch(
        `/orgs/${this.orgId}/crawlconfigs/${this.workflow!.id}/run`,
@ -1465,15 +1455,7 @@ export class WorkflowDetail extends LiteElement {
      let message = msg("Sorry, couldn't run crawl at this time.");
      if (e.isApiError && e.statusCode === 403) {
        if (e.details === "storage_quota_reached") {
-          message = msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-          );
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: true },
-              bubbles: true,
-            })
-          );
+          message = msg("Your org does not have enough storage to run crawls.");
        } else {
          message = msg("You do not have permission to run crawls.");
        }
--- a/frontend/src/pages/org/workflow-editor.ts
+++ b/frontend/src/pages/org/workflow-editor.ts
@ -2037,27 +2037,28 @@ https://archiveweb.page/images/${"logo.svg"}`}
      const crawlId = data.run_now_job;
      const storageQuotaReached = data.storageQuotaReached;

-      let message = msg("Workflow created.");
-      if (crawlId && !storageQuotaReached) {
-        message = msg("Crawl started with new template.");
-      } else if (this.configId) {
-        message = msg("Workflow updated.");
-      }
-
-      this.notify({
-        message,
-        variant: "success",
-        icon: "check2-circle",
-        duration: 8000,
-      });
-
-      if (storageQuotaReached) {
+      if (crawlId && storageQuotaReached) {
        this.notify({
+          title: msg("Workflow saved."),
          message: msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
+            "Could not start crawl with new workflow settings due to storage quota."
          ),
-          variant: "danger",
-          icon: "exclamation-octagon",
+          variant: "warning",
+          icon: "exclamation-triangle",
+          duration: 8000,
+        });
+      } else {
+        let message = msg("Workflow created.");
+        if (crawlId) {
+          message = msg("Crawl started with new workflow settings.");
+        } else if (this.configId) {
+          message = msg("Workflow updated.");
+        }
+
+        this.notify({
+          message,
+          variant: "success",
+          icon: "check2-circle",
        });
      }

--- a/frontend/src/pages/org/workflows-list.ts
+++ b/frontend/src/pages/org/workflows-list.ts
@ -199,21 +199,15 @@ export class WorkflowsList extends LiteElement {
          ${when(
            this.isCrawler,
            () => html`
-              <sl-tooltip
-                content=${msg("Org Storage Full")}
-                ?disabled=${!this.orgStorageQuotaReached}
+              <sl-button
+                href=${`/orgs/${this.orgId}/workflows?new&jobType=`}
+                variant="primary"
+                size="small"
+                @click=${this.navLink}
              >
-                <sl-button
-                  href=${`/orgs/${this.orgId}/workflows?new&jobType=`}
-                  variant="primary"
-                  size="small"
-                  ?disabled=${this.orgStorageQuotaReached}
-                  @click=${this.navLink}
-                >
-                  <sl-icon slot="prefix" name="plus-lg"></sl-icon>
-                  ${msg("New Workflow")}
-                </sl-button>
-              </sl-tooltip>
+                <sl-icon slot="prefix" name="plus-lg"></sl-icon>
+                ${msg("New Workflow")}
+              </sl-button>
            `
          )}
        </div>
@ -434,6 +428,7 @@ export class WorkflowsList extends LiteElement {
        () => html`
          <sl-menu-item
            style="--sl-color-neutral-700: var(--success)"
+            ?disabled=${this.orgStorageQuotaReached}
            @click=${() => this.runNow(workflow)}
          >
            <sl-icon name="play" slot="prefix"></sl-icon>
@ -745,17 +740,6 @@ export class WorkflowsList extends LiteElement {
  }

  private async runNow(workflow: Workflow): Promise<void> {
-    if (this.orgStorageQuotaReached) {
-      this.notify({
-        message: msg(
-          "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-        ),
-        variant: "danger",
-        icon: "exclamation-octagon",
-      });
-      return;
-    }
-
    try {
      const data = await this.apiFetch(
        `/orgs/${this.orgId}/crawlconfigs/${workflow.id}/run`,
@ -788,15 +772,7 @@ export class WorkflowsList extends LiteElement {
      let message = msg("Sorry, couldn't run crawl at this time.");
      if (e.isApiError && e.statusCode === 403) {
        if (e.details === "storage_quota_reached") {
-          message = msg(
-            "The org has reached its storage limit. Delete any archived items that are unneeded to free up space, or contact us to purchase a plan with more storage."
-          );
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: true },
-              bubbles: true,
-            })
-          );
+          message = msg("Your org does not have enough storage to run crawls.");
        } else {
          message = msg("You do not have permission to run crawls.");
        }
--- a/frontend/src/utils/LiteElement.ts
+++ b/frontend/src/utils/LiteElement.ts
@ -123,21 +123,55 @@ export default class LiteElement extends LitElement {
      ...opts,
    });

-    if (resp.status !== 200) {
-      if (resp.status === 401) {
-        this.dispatchEvent(new CustomEvent("need-login"));
+    if (resp.ok) {
+      const body = await resp.json();
+      const storageQuotaReached = body.storageQuotaReached;
+      if (typeof storageQuotaReached === "boolean") {
+        this.dispatchEvent(
+          new CustomEvent("storage-quota-update", {
+            detail: { reached: storageQuotaReached },
+            bubbles: true,
+          })
+        );
      }

-      let detail;
-      let errorMessage: string = msg("Unknown API error");
+      return body;
+    }

-      try {
-        detail = (await resp.json()).detail;
+    let errorDetail;
+    try {
+      errorDetail = (await resp.json()).detail;
+    } catch {}

-        if (typeof detail === "string") {
-          errorMessage = detail;
-        } else if (Array.isArray(detail) && detail.length) {
-          const fieldDetail = detail[0];
+    let errorMessage: string = msg("Unknown API error");
+
+    switch (resp.status) {
+      case 401: {
+        this.dispatchEvent(new CustomEvent("need-login"));
+        errorMessage = msg("Need login");
+        break;
+      }
+      case 403: {
+        if (errorDetail === "storage_quota_reached") {
+          this.dispatchEvent(
+            new CustomEvent("storage-quota-update", {
+              detail: { reached: true },
+              bubbles: true,
+            })
+          );
+          errorMessage = msg("Storage quota reached");
+          break;
+        }
+      }
+      case 404: {
+        errorMessage = msg("Not found");
+        break;
+      }
+      default: {
+        if (typeof errorDetail === "string") {
+          errorMessage = errorDetail;
+        } else if (Array.isArray(errorDetail) && errorDetail.length) {
+          const fieldDetail = errorDetail[0] || {};
          const { loc, msg } = fieldDetail;

          const fieldName = loc
@ -145,31 +179,14 @@ export default class LiteElement extends LitElement {
            .join(" ");
          errorMessage = `${fieldName} ${msg}`;
        }
-      } catch {}
-
-      throw new APIError({
-        message: errorMessage,
-        status: resp.status,
-        details: detail,
-      });
+        break;
+      }
    }

-    const body = await resp.json();
-
-    if (options?.method && options?.method !== "GET" && resp.status === 200) {
-      try {
-        const storageQuotaReached = body.storageQuotaReached;
-        if (typeof storageQuotaReached === "boolean") {
-          this.dispatchEvent(
-            new CustomEvent("storage-quota-update", {
-              detail: { reached: storageQuotaReached },
-              bubbles: true,
-            })
-          );
-        }
-      } catch {}
-    }
-
-    return await body;
+    throw new APIError({
+      message: errorMessage,
+      status: resp.status,
+      details: errorDetail,
+    });
  }
 }