19879fe349
- Add default vs custom (s3) storage - K8S: All storages correspond to secrets - K8S: Default storages inited via helm - K8S: Custom storage results in custom secret (per archive) - K8S: Don't add secret per crawl config - API for changing storage per archive - Docker: default storage just hard-coded from env vars (only one for now) - Validate custom storage via aiobotocore before confirming - Data Model: remove usage from users - Data Model: support adding multiple files per crawl for parallel crawls - Data Model: track completions for parallel crawls - Data Model: initial support for tags per crawl, add collection as 'coll' tag README fixes
53 lines
1.2 KiB
Python
53 lines
1.2 KiB
Python
"""
|
|
Browsertrix API Mongo DB initialization
|
|
"""
|
|
|
|
import os
|
|
from typing import Optional
|
|
|
|
import motor.motor_asyncio
|
|
|
|
from pydantic import BaseModel
|
|
|
|
|
|
DATABASE_URL = (
|
|
f"mongodb://root:example@{os.environ.get('MONGO_HOST', 'localhost')}:27017"
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
def init_db():
|
|
"""initializde the mongodb connector"""
|
|
client = motor.motor_asyncio.AsyncIOMotorClient(
|
|
DATABASE_URL, uuidRepresentation="standard"
|
|
)
|
|
|
|
mdb = client["browsertrixcloud"]
|
|
|
|
return mdb
|
|
|
|
|
|
# ============================================================================
|
|
class BaseMongoModel(BaseModel):
|
|
"""Base pydantic model that is also a mongo doc"""
|
|
|
|
id: Optional[str]
|
|
|
|
@classmethod
|
|
def from_dict(cls, data):
|
|
"""convert dict from mongo to an Archive"""
|
|
if not data:
|
|
return None
|
|
data["id"] = str(data.pop("_id"))
|
|
return cls(**data)
|
|
|
|
def serialize(self):
|
|
"""convert Archive to dict"""
|
|
return self.dict(exclude_unset=True, exclude_defaults=True, exclude_none=True)
|
|
|
|
def to_dict(self, **opts):
|
|
"""convert to dict for mongo"""
|
|
res = self.dict(**opts)
|
|
res["_id"] = res.pop("id", "")
|
|
return res
|