diff --git a/backend/btrixcloud/background_jobs.py b/backend/btrixcloud/background_jobs.py index d0e91410..7490229a 100644 --- a/backend/btrixcloud/background_jobs.py +++ b/backend/btrixcloud/background_jobs.py @@ -1,4 +1,5 @@ """k8s background jobs""" +import asyncio from datetime import datetime from typing import Optional, Tuple, Union, List, Dict, TYPE_CHECKING, cast from uuid import UUID @@ -31,6 +32,7 @@ else: # ============================================================================ +# pylint: disable=too-many-instance-attributes class BackgroundJobOps: """k8s background job management""" @@ -43,9 +45,12 @@ class BackgroundJobOps: # pylint: disable=too-many-locals, too-many-arguments, invalid-name - def __init__(self, mdb, org_ops, crawl_manager, storage_ops): + def __init__(self, mdb, email, user_manager, org_ops, crawl_manager, storage_ops): self.jobs = mdb["jobs"] + self.email = email + self.user_manager = user_manager + self.org_ops = org_ops self.crawl_manager = crawl_manager self.storage_ops = storage_ops @@ -215,6 +220,21 @@ class BackgroundJobOps: if success: if job_type == BgJobType.CREATE_REPLICA: await self.handle_replica_job_finished(cast(CreateReplicaJob, job)) + else: + print( + f"Background job {job.id} failed, sending email to superuser", + flush=True, + ) + superuser = await self.user_manager.get_superuser() + org = await self.org_ops.get_org_by_id(job.oid) + await asyncio.get_event_loop().run_in_executor( + None, + self.email.send_background_job_failed, + job, + org, + finished, + superuser.email, + ) await self.jobs.find_one_and_update( {"_id": job_id, "oid": oid}, @@ -307,11 +327,15 @@ class BackgroundJobOps: # ============================================================================ # pylint: disable=too-many-arguments, too-many-locals, invalid-name, fixme -def init_background_jobs_api(mdb, org_ops, crawl_manager, storage_ops): +def init_background_jobs_api( + mdb, email, user_manager, org_ops, crawl_manager, storage_ops +): """init background jobs system""" # pylint: disable=invalid-name - ops = BackgroundJobOps(mdb, org_ops, crawl_manager, storage_ops) + ops = BackgroundJobOps( + mdb, email, user_manager, org_ops, crawl_manager, storage_ops + ) router = ops.router diff --git a/backend/btrixcloud/emailsender.py b/backend/btrixcloud/emailsender.py index 40273064..3b7c6c6d 100644 --- a/backend/btrixcloud/emailsender.py +++ b/backend/btrixcloud/emailsender.py @@ -1,11 +1,14 @@ """ Basic Email Sending Support""" +from datetime import datetime import os import smtplib import ssl -from typing import Optional +from typing import Optional, Union from email.message import EmailMessage + +from .models import CreateReplicaJob, DeleteReplicaJob, Organization from .utils import is_bool @@ -137,3 +140,30 @@ to create a new password """ self._send_encrypted(receiver_email, "Password Reset", message) + + def send_background_job_failed( + self, + job: Union[CreateReplicaJob, DeleteReplicaJob], + org: Organization, + finished: datetime, + receiver_email: str, + ): + """Send background job failed email to superuser""" + message = f""" +Failed Background Job +--------------------- + +Organization: {org.name} ({job.oid}) +Job type: {job.type} + +Job ID: {job.id} +Started: {job.started.isoformat(sep=" ", timespec="seconds")}Z +Finished: {finished.isoformat(sep=" ", timespec="seconds")}Z + +Object type: {job.object_type} +Object ID: {job.object_id} +File path: {job.file_path} +Replica storage name: {job.replica_storage.name} + """ + + self._send_encrypted(receiver_email, "Failed Background Job", message) diff --git a/backend/btrixcloud/main.py b/backend/btrixcloud/main.py index a01a37e2..b93eddf6 100644 --- a/backend/btrixcloud/main.py +++ b/backend/btrixcloud/main.py @@ -90,7 +90,7 @@ def main(): storage_ops = init_storages_api(org_ops, crawl_manager) background_job_ops = init_background_jobs_api( - mdb, org_ops, crawl_manager, storage_ops + mdb, email, user_manager, org_ops, crawl_manager, storage_ops ) profiles = init_profiles_api( diff --git a/backend/btrixcloud/main_op.py b/backend/btrixcloud/main_op.py index 579ac63b..1bfebed2 100644 --- a/backend/btrixcloud/main_op.py +++ b/backend/btrixcloud/main_op.py @@ -54,7 +54,9 @@ def main(): storage_ops = init_storages_api(org_ops, crawl_manager) - background_job_ops = BackgroundJobOps(mdb, org_ops, crawl_manager, storage_ops) + background_job_ops = BackgroundJobOps( + mdb, email, user_manager, org_ops, crawl_manager, storage_ops + ) profile_ops = ProfileOps( mdb, org_ops, crawl_manager, storage_ops, background_job_ops