From b08a188fea0dc2ae637417907cbc07b061f0362f Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Mon, 28 Jun 2021 15:48:59 -0700 Subject: [PATCH] initial commit! --- .gitignore | 2 + backend/Dockerfile | 12 ++++++ backend/archives.py | 55 ++++++++++++++++++++++++++ backend/crawls.py | 17 ++++++++ backend/db.py | 17 ++++++++ backend/dockerdriver.py | 9 +++++ backend/main.py | 31 +++++++++++++++ backend/requirements.txt | 3 ++ backend/users.py | 84 ++++++++++++++++++++++++++++++++++++++++ docker-compose.yaml | 35 +++++++++++++++++ update.sh | 1 + 11 files changed, 266 insertions(+) create mode 100644 .gitignore create mode 100644 backend/Dockerfile create mode 100644 backend/archives.py create mode 100644 backend/crawls.py create mode 100644 backend/db.py create mode 100644 backend/dockerdriver.py create mode 100644 backend/main.py create mode 100644 backend/requirements.txt create mode 100644 backend/users.py create mode 100644 docker-compose.yaml create mode 100755 update.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..24f9ca98 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +**/*.pyc +**/node_modules/ diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..edf9a442 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.9 + +WORKDIR /app + +ADD requirements.txt /app + +RUN pip install -r requirements.txt + +ADD . /app + +CMD uvicorn main:app --host 0.0.0.0 --reload --access-log --log-level debug + diff --git a/backend/archives.py b/backend/archives.py new file mode 100644 index 00000000..ee31ae0b --- /dev/null +++ b/backend/archives.py @@ -0,0 +1,55 @@ +from typing import List, Optional, TypeVar +from pydantic import BaseModel, UUID4, validator +from fastapi import APIRouter, Depends +from users import User +import uuid +from bson.objectid import ObjectId + +class Archive(BaseModel): + #id: Optional[UUID4] + title: Optional[str] + user: Optional[UUID4] + + +class S3Archive(Archive): + endpoint_url: Optional[str] + is_public: Optional[bool] + + #@validator("id", pre=True, always=True) + #def default_id(cls, v): + # return v or uuid.uuid4() + + +def init_archives_api(app, db, user_dep: User): + archives_coll = db["archives"] + + router = APIRouter( + prefix="/archives", + tags=["archives"], + responses={404: {"description": "Not found"}}, + ) + + @router.get("/") + async def get_archives(user: User=Depends(user_dep)): + cursor = archives_coll.find({}) + results = await cursor.to_list(length=1000) + return {"archives": [{"id": str(res["_id"]), "title": res["title"], "endpoint_url": res["endpoint_url"]} for res in results]} + + @router.get("/{id}") + async def get_archives(id: str, user: User=Depends(user_dep)): + res = await archives_coll.find_one(ObjectId(id)) + print(res) + if not res: + return {} + + return {"id": id, "title": res["title"], "endpoint_url": res["endpoint_url"]} + + @router.post("/") + async def add_archive(archive: S3Archive, user: User = Depends(user_dep)): + archive.user = user.id + print(archive.user) + res = await archives_coll.insert_one(archive.dict()) + return {"added": str(res.inserted_id)} + + app.include_router(router) + diff --git a/backend/crawls.py b/backend/crawls.py new file mode 100644 index 00000000..acc234a6 --- /dev/null +++ b/backend/crawls.py @@ -0,0 +1,17 @@ +from typing import List, Optional, TypeVar +from pydantic import BaseModel, UUID4, validator +from fastapi import APIRouter, Depends +from users import User +import uuid +from bson.objectid import ObjectId + +class SimpleCrawl(BaseModel): + url: str + scopeType: str + + + + + + + diff --git a/backend/db.py b/backend/db.py new file mode 100644 index 00000000..1821dd1b --- /dev/null +++ b/backend/db.py @@ -0,0 +1,17 @@ +import os +import motor.motor_asyncio + + +DATABASE_URL = ( + f"mongodb://root:example@{os.environ.get('MONGO_HOST', 'localhost')}:27017" +) + + +def init_db(): + client = motor.motor_asyncio.AsyncIOMotorClient( + DATABASE_URL, uuidRepresentation="standard" + ) + + db = client["browsertrixcloud"] + + return db diff --git a/backend/dockerdriver.py b/backend/dockerdriver.py new file mode 100644 index 00000000..4ad24938 --- /dev/null +++ b/backend/dockerdriver.py @@ -0,0 +1,9 @@ +import aiodocker + +class DockerDriver(BaseDriver): + def __init__(self): + self.docker = aiodocker.Docker() + self.crawl_image = os.environ.get("CRAWLER_IMAGE", "webrecorder/browsertrix-crawler") + + def start_crawl(self): + container = await self.docker.containers.create(config=config) diff --git a/backend/main.py b/backend/main.py new file mode 100644 index 00000000..5af484d4 --- /dev/null +++ b/backend/main.py @@ -0,0 +1,31 @@ +from fastapi import FastAPI, Depends + +import logging +import os +import sys +import json + +from users import init_users_api, User +from db import init_db +from archives import init_archives_api + + +db = init_db() + +app = FastAPI() + +fastapi_users = init_users_api(app, db) + +current_active_user = fastapi_users.current_user(active=True) + +init_archives_api(app, db, current_active_user) + + +@app.get("/") +async def root(): + return {"message": "Hello World"} + + +@app.get("/protected-route") +def protected_route(user: User = Depends(current_active_user)): + return f"Hello, {user.email}" diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 00000000..eeac9d4e --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,3 @@ +uvicorn +fastapi-users[mongodb]==6.0.0 +loguru diff --git a/backend/users.py b/backend/users.py new file mode 100644 index 00000000..85e07a7a --- /dev/null +++ b/backend/users.py @@ -0,0 +1,84 @@ +import os +import uuid +from fastapi import Request +from fastapi_users import FastAPIUsers, models +from fastapi_users.authentication import JWTAuthentication +from fastapi_users.db import MongoDBUserDatabase + +PASSWORD_SECRET = os.environ.get("PASSWORD_SECRET", uuid.uuid4().hex) + + +class User(models.BaseUser): + pass + + +class UserCreate(models.BaseUserCreate): + pass + + +class UserUpdate(User, models.BaseUserUpdate): + pass + + +class UserDB(User, models.BaseUserDB): + pass + + +def on_after_register(user: UserDB, request: Request): + print(f"User {user.id} has registered.") + + +def on_after_forgot_password(user: UserDB, token: str, request: Request): + print(f"User {user.id} has forgot their password. Reset token: {token}") + + +def after_verification_request(user: UserDB, token: str, request: Request): + print(f"Verification requested for user {user.id}. Verification token: {token}") + + +def init_users_api(app, db): + user_collection = db["users"] + + user_db = MongoDBUserDatabase(UserDB, user_collection) + + jwt_authentication = JWTAuthentication( + secret=PASSWORD_SECRET, lifetime_seconds=3600, tokenUrl="/auth/jwt/login" + ) + + fastapi_users = FastAPIUsers( + user_db, + [jwt_authentication], + User, + UserCreate, + UserUpdate, + UserDB, + ) + app.include_router( + fastapi_users.get_auth_router(jwt_authentication), + prefix="/auth/jwt", + tags=["auth"], + ) + app.include_router( + fastapi_users.get_register_router(on_after_register), + prefix="/auth", + tags=["auth"], + ) + app.include_router( + fastapi_users.get_reset_password_router( + PASSWORD_SECRET, after_forgot_password=on_after_forgot_password + ), + prefix="/auth", + tags=["auth"], + ) + app.include_router( + fastapi_users.get_verify_router( + PASSWORD_SECRET, after_verification_request=after_verification_request + ), + prefix="/auth", + tags=["auth"], + ) + app.include_router( + fastapi_users.get_users_router(), prefix="/users", tags=["users"] + ) + + return fastapi_users diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 00000000..4db9d4f0 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,35 @@ +version: '3.5' + +services: + backend: + build: ./backend + image: btrixcloud/backend + ports: + - 8000:8000 + + environment: + MONGO_HOST: mongo + PASSWORD_SECRET: 'c9085f33ecce4347aa1d69339e16c499' + + mongo: + image: mongo + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: example + + volumes: + - mongodata:/data/db + + minio: + image: minio/minio + command: server /data + ports: + - 8010:9000 + + volumes: + - miniodata:/data + +volumes: + mongodata: + miniodata: + diff --git a/update.sh b/update.sh new file mode 100755 index 00000000..21fcfdef --- /dev/null +++ b/update.sh @@ -0,0 +1 @@ +docker-compose build; docker-compose kill; docker-compose rm -f; docker-compose up -d