Wait for DB init for healthcheck + settings (#885)

* init check: (backend fix for #794)
- wait until db is inited before settings /api/settings to return 200
- also return 503 from healthcheck endpoint, until db is available

backend/btrixcloud/db.py

@@ -4,6 +4,8 @@ Browsertrix API Mongo DB initialization
 import importlib.util
 import os
 import urllib
+import asyncio
+
 from typing import Optional
 
 import motor.motor_asyncio
@@ -49,6 +51,23 @@ def init_db():
     return client, mdb
 
 
+# ============================================================================
+async def ping_db(mdb, db_inited):
+    """run in loop until db is up, set db_inited['inited'] property to true"""
+    print("Waiting DB", flush=True)
+    while True:
+        try:
+            result = await mdb.command("ping")
+            assert result.get("ok")
+            db_inited["inited"] = True
+            print("DB Ready!")
+            break
+        # pylint: disable=broad-exception-caught
+        except Exception:
+            print("Retrying, waiting for DB to be ready")
+            await asyncio.sleep(3)
+
+
 # ============================================================================
 async def update_and_prepare_db(
     # pylint: disable=R0913
@@ -59,6 +78,7 @@ async def update_and_prepare_db(
     crawl_config_ops,
     coll_ops,
     invite_ops,
+    db_inited,
 ):
     """Prepare database for application.
 
@@ -68,6 +88,7 @@ async def update_and_prepare_db(
     - Create/update default org
 
     """
+    await ping_db(mdb, db_inited)
     print("Database setup started", flush=True)
     if await run_db_migrations(mdb, user_manager):
         await drop_indexes(mdb)

backend/btrixcloud/main.py

@@ -6,11 +6,11 @@ import os
 import asyncio
 import sys
 
-from fastapi import FastAPI
+from fastapi import FastAPI, HTTPException
 from fastapi.responses import JSONResponse
 from fastapi.routing import APIRouter
 
-from .db import init_db, update_and_prepare_db
+from .db import init_db, ping_db, update_and_prepare_db
 
 from .emailsender import EmailSender
 from .invites import init_invites
@@ -35,6 +35,8 @@ app_root = FastAPI(
     openapi_url=API_PREFIX + "/openapi.json",
 )
 
+db_inited = {"inited": False}
+
 
 # ============================================================================
 # pylint: disable=too-many-locals
@@ -114,14 +116,25 @@ def main():
     if run_once_lock("btrix-init-db"):
         asyncio.create_task(
             update_and_prepare_db(
-                mdb, user_manager, org_ops, crawls, crawl_config_ops, coll_ops, invites
+                mdb,
+                user_manager,
+                org_ops,
+                crawls,
+                crawl_config_ops,
+                coll_ops,
+                invites,
+                db_inited,
             )
         )
+    else:
+        asyncio.create_task(ping_db(mdb, db_inited))
 
     app.include_router(org_ops.router)
 
     @app.get("/settings")
     async def get_settings():
+        if not db_inited.get("inited"):
+            raise HTTPException(status_code=503, detail="not_ready_yet")
         return settings
 
     # internal routes
@@ -132,6 +145,8 @@ def main():
 
     @app_root.get("/healthz", include_in_schema=False)
     async def healthz():
+        if not db_inited.get("inited"):
+            raise HTTPException(status_code=503, detail="not_ready_yet")
         return {}
 
     app_root.include_router(app, prefix=API_PREFIX)