fa86555eed
* keep track of per pod status on crawljob: - crashes time, and reason - 'used' vs 'allocated' resources - 'percent' used / allocated * crawl log errors: log error when crawler crashes via OOM, either via redis error log or to console * add initial autoscaling support! - detect if metrics server is available via K8SApi.is_pod_metrics_available() - if available, use metrics for 'used' fields - if no metrics, set memory used for redis only (using redis apis) - allow overriding memory and cpu via newMemory and newCpu settings on pod status - scale memory / cpu based on newMemory and newCpu setting - templates: update jinja templates to allow restarting crawler and redis with new resources - ci: enable metrics-server on k3d, microk8s and nightly k3d ci runs * roles: cleanup unused roles, add permissions for listing metrics * stats for running crawls: - update in db via operator - avoids losing stats if redis pod happens to be done - tradeoff is more db access in operator, but less extra connections to redis + already loading from db in backend - size stat: ensure size of previous files is added to the stats * crawler deployment tweaks: - adjust cpu/mem per browser - add --headless flag to configmap to use new headless mode by default!
90 lines
2.2 KiB
Python
90 lines
2.2 KiB
Python
""" entrypoint module for operator """
|
|
|
|
import os
|
|
import sys
|
|
|
|
from fastapi import FastAPI
|
|
|
|
from .crawlmanager import CrawlManager
|
|
from .db import init_db
|
|
from .emailsender import EmailSender
|
|
from .operator import init_operator_api
|
|
from .utils import register_exit_handler
|
|
|
|
from .invites import InviteOps
|
|
from .users import init_user_manager
|
|
from .orgs import OrgOps
|
|
from .colls import CollectionOps
|
|
from .crawlconfigs import CrawlConfigOps
|
|
from .crawls import CrawlOps
|
|
from .profiles import ProfileOps
|
|
from .webhooks import EventWebhookOps
|
|
|
|
app_root = FastAPI()
|
|
|
|
|
|
# ============================================================================
|
|
# pylint: disable=too-many-function-args, duplicate-code
|
|
def main():
|
|
"""main init"""
|
|
email = EmailSender()
|
|
crawl_manager = None
|
|
|
|
dbclient, mdb = init_db()
|
|
|
|
invite_ops = InviteOps(mdb, email)
|
|
|
|
user_manager = init_user_manager(mdb, email, invite_ops)
|
|
|
|
org_ops = OrgOps(mdb, invite_ops)
|
|
|
|
event_webhook_ops = EventWebhookOps(mdb, org_ops)
|
|
|
|
user_manager.set_org_ops(org_ops)
|
|
|
|
# pylint: disable=import-outside-toplevel
|
|
if not os.environ.get("KUBERNETES_SERVICE_HOST"):
|
|
print(
|
|
"Sorry, the Browsertrix Cloud Backend must be run inside a Kubernetes environment.\
|
|
Kubernetes not detected (KUBERNETES_SERVICE_HOST is not set), Exiting"
|
|
)
|
|
sys.exit(1)
|
|
|
|
crawl_manager = CrawlManager()
|
|
|
|
profile_ops = ProfileOps(mdb, org_ops, crawl_manager)
|
|
|
|
crawl_config_ops = CrawlConfigOps(
|
|
dbclient,
|
|
mdb,
|
|
user_manager,
|
|
org_ops,
|
|
crawl_manager,
|
|
profile_ops,
|
|
)
|
|
|
|
coll_ops = CollectionOps(mdb, crawl_manager, org_ops, event_webhook_ops)
|
|
|
|
crawl_ops = CrawlOps(
|
|
mdb,
|
|
user_manager,
|
|
crawl_manager,
|
|
crawl_config_ops,
|
|
org_ops,
|
|
coll_ops,
|
|
event_webhook_ops,
|
|
)
|
|
|
|
return init_operator_api(
|
|
app_root, crawl_config_ops, crawl_ops, org_ops, coll_ops, event_webhook_ops
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
@app_root.on_event("startup")
|
|
async def startup():
|
|
"""init on startup"""
|
|
register_exit_handler()
|
|
oper = main()
|
|
await oper.async_init()
|