initial commit!

.gitignore

@@ -0,0 +1,2 @@
+**/*.pyc
+**/node_modules/

backend/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.9
+
+WORKDIR /app
+
+ADD requirements.txt /app
+
+RUN pip install -r requirements.txt
+
+ADD . /app
+
+CMD uvicorn main:app --host 0.0.0.0 --reload --access-log --log-level debug
+

backend/archives.py

@@ -0,0 +1,55 @@
+from typing import List, Optional, TypeVar
+from pydantic import BaseModel, UUID4, validator
+from fastapi import APIRouter, Depends
+from users import User
+import uuid
+from bson.objectid import ObjectId
+
+class Archive(BaseModel):
+    #id: Optional[UUID4]
+    title: Optional[str]
+    user: Optional[UUID4]
+
+
+class S3Archive(Archive):
+    endpoint_url: Optional[str]
+    is_public: Optional[bool]
+
+    #@validator("id", pre=True, always=True)
+    #def default_id(cls, v):
+    #    return v or uuid.uuid4()
+
+
+def init_archives_api(app, db, user_dep: User):
+    archives_coll = db["archives"]
+
+    router = APIRouter(
+        prefix="/archives",
+        tags=["archives"],
+        responses={404: {"description": "Not found"}},
+    )
+
+    @router.get("/")
+    async def get_archives(user: User=Depends(user_dep)):
+        cursor = archives_coll.find({})
+        results = await cursor.to_list(length=1000)
+        return {"archives": [{"id": str(res["_id"]), "title": res["title"], "endpoint_url": res["endpoint_url"]} for res in results]}
+
+    @router.get("/{id}")
+    async def get_archives(id: str, user: User=Depends(user_dep)):
+        res = await archives_coll.find_one(ObjectId(id))
+        print(res)
+        if not res:
+            return {}
+
+        return {"id": id, "title": res["title"], "endpoint_url": res["endpoint_url"]}
+
+    @router.post("/")
+    async def add_archive(archive: S3Archive, user: User = Depends(user_dep)):
+        archive.user = user.id
+        print(archive.user)
+        res = await archives_coll.insert_one(archive.dict())
+        return {"added": str(res.inserted_id)}
+
+    app.include_router(router)
+

backend/crawls.py

@@ -0,0 +1,17 @@
+from typing import List, Optional, TypeVar
+from pydantic import BaseModel, UUID4, validator
+from fastapi import APIRouter, Depends
+from users import User
+import uuid
+from bson.objectid import ObjectId
+
+class SimpleCrawl(BaseModel):
+    url: str
+    scopeType: str
+
+
+
+
+
+
+

backend/db.py

@@ -0,0 +1,17 @@
+import os
+import motor.motor_asyncio
+
+
+DATABASE_URL = (
+    f"mongodb://root:example@{os.environ.get('MONGO_HOST', 'localhost')}:27017"
+)
+
+
+def init_db():
+    client = motor.motor_asyncio.AsyncIOMotorClient(
+        DATABASE_URL, uuidRepresentation="standard"
+    )
+
+    db = client["browsertrixcloud"]
+
+    return db

backend/dockerdriver.py

@@ -0,0 +1,9 @@
+import aiodocker
+
+class DockerDriver(BaseDriver):
+    def __init__(self):
+        self.docker = aiodocker.Docker()
+        self.crawl_image = os.environ.get("CRAWLER_IMAGE", "webrecorder/browsertrix-crawler")
+
+    def start_crawl(self):
+        container = await self.docker.containers.create(config=config)

backend/main.py

@@ -0,0 +1,31 @@
+from fastapi import FastAPI, Depends
+
+import logging
+import os
+import sys
+import json
+
+from users import init_users_api, User
+from db import init_db
+from archives import init_archives_api
+
+
+db = init_db()
+
+app = FastAPI()
+
+fastapi_users = init_users_api(app, db)
+
+current_active_user = fastapi_users.current_user(active=True)
+
+init_archives_api(app, db, current_active_user)
+
+
+@app.get("/")
+async def root():
+    return {"message": "Hello World"}
+
+
+@app.get("/protected-route")
+def protected_route(user: User = Depends(current_active_user)):
+    return f"Hello, {user.email}"

backend/requirements.txt

@@ -0,0 +1,3 @@
+uvicorn
+fastapi-users[mongodb]==6.0.0
+loguru

backend/users.py

@@ -0,0 +1,84 @@
+import os
+import uuid
+from fastapi import Request
+from fastapi_users import FastAPIUsers, models
+from fastapi_users.authentication import JWTAuthentication
+from fastapi_users.db import MongoDBUserDatabase
+
+PASSWORD_SECRET = os.environ.get("PASSWORD_SECRET", uuid.uuid4().hex)
+
+
+class User(models.BaseUser):
+    pass
+
+
+class UserCreate(models.BaseUserCreate):
+    pass
+
+
+class UserUpdate(User, models.BaseUserUpdate):
+    pass
+
+
+class UserDB(User, models.BaseUserDB):
+    pass
+
+
+def on_after_register(user: UserDB, request: Request):
+    print(f"User {user.id} has registered.")
+
+
+def on_after_forgot_password(user: UserDB, token: str, request: Request):
+    print(f"User {user.id} has forgot their password. Reset token: {token}")
+
+
+def after_verification_request(user: UserDB, token: str, request: Request):
+    print(f"Verification requested for user {user.id}. Verification token: {token}")
+
+
+def init_users_api(app, db):
+    user_collection = db["users"]
+
+    user_db = MongoDBUserDatabase(UserDB, user_collection)
+
+    jwt_authentication = JWTAuthentication(
+        secret=PASSWORD_SECRET, lifetime_seconds=3600, tokenUrl="/auth/jwt/login"
+    )
+
+    fastapi_users = FastAPIUsers(
+        user_db,
+        [jwt_authentication],
+        User,
+        UserCreate,
+        UserUpdate,
+        UserDB,
+    )
+    app.include_router(
+        fastapi_users.get_auth_router(jwt_authentication),
+        prefix="/auth/jwt",
+        tags=["auth"],
+    )
+    app.include_router(
+        fastapi_users.get_register_router(on_after_register),
+        prefix="/auth",
+        tags=["auth"],
+    )
+    app.include_router(
+        fastapi_users.get_reset_password_router(
+            PASSWORD_SECRET, after_forgot_password=on_after_forgot_password
+        ),
+        prefix="/auth",
+        tags=["auth"],
+    )
+    app.include_router(
+        fastapi_users.get_verify_router(
+            PASSWORD_SECRET, after_verification_request=after_verification_request
+        ),
+        prefix="/auth",
+        tags=["auth"],
+    )
+    app.include_router(
+        fastapi_users.get_users_router(), prefix="/users", tags=["users"]
+    )
+
+    return fastapi_users

docker-compose.yaml

@@ -0,0 +1,35 @@
+version: '3.5'
+
+services:
+  backend:
+    build: ./backend
+    image: btrixcloud/backend
+    ports:
+      - 8000:8000
+
+    environment:
+      MONGO_HOST: mongo
+      PASSWORD_SECRET: 'c9085f33ecce4347aa1d69339e16c499'
+ 
+  mongo:
+    image: mongo
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: example
+
+    volumes:
+      - mongodata:/data/db
+
+  minio:
+    image: minio/minio
+    command: server /data
+    ports:
+      - 8010:9000
+
+    volumes:
+      - miniodata:/data
+
+volumes:
+  mongodata:
+  miniodata:
+

update.sh

@@ -0,0 +1 @@
+docker-compose build; docker-compose kill; docker-compose rm -f; docker-compose up -d