Send email to superuser when background job fails (#1355)

Fixes #1344

Sends email to superadmin when a background job fails.

backend/btrixcloud/background_jobs.py

@@ -1,4 +1,5 @@
 """k8s background jobs"""
+import asyncio
 from datetime import datetime
 from typing import Optional, Tuple, Union, List, Dict, TYPE_CHECKING, cast
 from uuid import UUID
@@ -31,6 +32,7 @@ else:
 
 
 # ============================================================================
+# pylint: disable=too-many-instance-attributes
 class BackgroundJobOps:
     """k8s background job management"""
 
@@ -43,9 +45,12 @@ class BackgroundJobOps:
 
     # pylint: disable=too-many-locals, too-many-arguments, invalid-name
 
-    def __init__(self, mdb, org_ops, crawl_manager, storage_ops):
+    def __init__(self, mdb, email, user_manager, org_ops, crawl_manager, storage_ops):
         self.jobs = mdb["jobs"]
 
+        self.email = email
+        self.user_manager = user_manager
+
         self.org_ops = org_ops
         self.crawl_manager = crawl_manager
         self.storage_ops = storage_ops
@@ -215,6 +220,21 @@ class BackgroundJobOps:
         if success:
             if job_type == BgJobType.CREATE_REPLICA:
                 await self.handle_replica_job_finished(cast(CreateReplicaJob, job))
+        else:
+            print(
+                f"Background job {job.id} failed, sending email to superuser",
+                flush=True,
+            )
+            superuser = await self.user_manager.get_superuser()
+            org = await self.org_ops.get_org_by_id(job.oid)
+            await asyncio.get_event_loop().run_in_executor(
+                None,
+                self.email.send_background_job_failed,
+                job,
+                org,
+                finished,
+                superuser.email,
+            )
 
         await self.jobs.find_one_and_update(
             {"_id": job_id, "oid": oid},
@@ -307,11 +327,15 @@ class BackgroundJobOps:
 
 # ============================================================================
 # pylint: disable=too-many-arguments, too-many-locals, invalid-name, fixme
-def init_background_jobs_api(mdb, org_ops, crawl_manager, storage_ops):
+def init_background_jobs_api(
+    mdb, email, user_manager, org_ops, crawl_manager, storage_ops
+):
     """init background jobs system"""
     # pylint: disable=invalid-name
 
-    ops = BackgroundJobOps(mdb, org_ops, crawl_manager, storage_ops)
+    ops = BackgroundJobOps(
+        mdb, email, user_manager, org_ops, crawl_manager, storage_ops
+    )
 
     router = ops.router
 

backend/btrixcloud/emailsender.py

@@ -1,11 +1,14 @@
 """ Basic Email Sending Support"""
 
+from datetime import datetime
 import os
 import smtplib
 import ssl
-from typing import Optional
+from typing import Optional, Union
 
 from email.message import EmailMessage
+
+from .models import CreateReplicaJob, DeleteReplicaJob, Organization
 from .utils import is_bool
 
 
@@ -137,3 +140,30 @@ to create a new password
         """
 
         self._send_encrypted(receiver_email, "Password Reset", message)
+
+    def send_background_job_failed(
+        self,
+        job: Union[CreateReplicaJob, DeleteReplicaJob],
+        org: Organization,
+        finished: datetime,
+        receiver_email: str,
+    ):
+        """Send background job failed email to superuser"""
+        message = f"""
+Failed Background Job
+---------------------
+
+Organization: {org.name} ({job.oid})
+Job type: {job.type}
+
+Job ID: {job.id}
+Started: {job.started.isoformat(sep=" ", timespec="seconds")}Z
+Finished: {finished.isoformat(sep=" ", timespec="seconds")}Z
+
+Object type: {job.object_type}
+Object ID: {job.object_id}
+File path: {job.file_path}
+Replica storage name: {job.replica_storage.name}
+        """
+
+        self._send_encrypted(receiver_email, "Failed Background Job", message)

backend/btrixcloud/main.py

@@ -90,7 +90,7 @@ def main():
     storage_ops = init_storages_api(org_ops, crawl_manager)
 
     background_job_ops = init_background_jobs_api(
-        mdb, org_ops, crawl_manager, storage_ops
+        mdb, email, user_manager, org_ops, crawl_manager, storage_ops
     )
 
     profiles = init_profiles_api(

backend/btrixcloud/main_op.py

@@ -54,7 +54,9 @@ def main():
 
     storage_ops = init_storages_api(org_ops, crawl_manager)
 
-    background_job_ops = BackgroundJobOps(mdb, org_ops, crawl_manager, storage_ops)
+    background_job_ops = BackgroundJobOps(
+        mdb, email, user_manager, org_ops, crawl_manager, storage_ops
+    )
 
     profile_ops = ProfileOps(
         mdb, org_ops, crawl_manager, storage_ops, background_job_ops