tea/browsertrix
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support unified docker + k8s deployment (#58)

Browse Source 
- adapt nginx config to work both in docker and k8s, using env vars to set urls

backend: additional fixes:
- use env vars with nginx config
- fix settings api route
- when sending e-mail, use the Host header for verification urls when available
- prepare Dockerfile with full build from scratch in image, (disabled 'yarn install' for faster builds for now)
- fix accept invite api for existing user to /archives/accept-invite/{token}
This commit is contained in:
Ilya Kreymer 2021-12-05 13:02:26 -08:00 committed by GitHub
parent 87c5505c43
commit eaf8055063
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 167 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
backend/archives.py
View File

@ -7,7 +7,7 @@ from datetime import datetime
from typing import Dict, Union, Literal, Optional
from pydantic import BaseModel
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends, HTTPException, Request
from db import BaseMongoModel
@ -298,18 +298,24 @@ def init_archives_api(app, mdb, user_manager, invites, user_dep: User):
@router.post("/invite", tags=["invites"])
async def invite_user_to_archive(
invite: InviteToArchiveRequest,
request: Request,
archive: Archive = Depends(archive_owner_dep),
user: User = Depends(user_dep),
):
if await invites.invite_user(
invite, user, user_manager, archive=archive, allow_existing=True
invite,
user,
user_manager,
archive=archive,
allow_existing=True,
headers=request.headers,
):
return {"invited": "new_user"}
return {"invited": "existing_user"}
@app.get("/invite/accept/{token}", tags=["invites"])
@app.post("/archives/invite-accept/{token}", tags=["invites"])
async def accept_invite(token: str, user: User = Depends(user_dep)):
invite = invites.accept_user_invite(user, token)

46
backend/emailsender.py
View File

@ -14,7 +14,7 @@ class EmailSender:
self.password = os.environ.get("EMAIL_PASSWORD")
self.smtp_server = os.environ.get("EMAIL_SMTP_HOST")
self.host = os.environ.get("APP_ORIGIN")
self.default_origin = os.environ.get("APP_ORIGIN")
def _send_encrypted(self, receiver, message):
"""Send Encrypted SMTP Message"""
@ -32,47 +32,71 @@ class EmailSender:
server.login(self.sender, self.password)
server.sendmail(self.sender, receiver, message)
def send_user_validation(self, receiver_email, token):
def get_origin(self, headers):
""" Return origin of the received request"""
scheme = headers.get("X-Forwarded-Proto")
if not scheme:
scheme = "http"
host = headers.get("Host")
if not host:
host = self.default_origin
return scheme + "://" + host
def send_user_validation(self, receiver_email, token, headers=None):
"""Send email to validate registration email address"""
origin = self.get_origin(headers)
message = f"""
Please verify your registration for Browsertrix Cloud for {receiver_email}
You can verify by clicking here: {self.host}/verify?token={token}
You can verify by clicking here: {origin}/verify?token={token}
The verification token is: {token}"""
self._send_encrypted(receiver_email, message)
def send_new_user_invite(self, receiver_email, sender, archive_name, token):
# pylint: disable=too-many-arguments
def send_new_user_invite(
self, receiver_email, sender, archive_name, token, headers=None
):
"""Send email to invite new user"""
origin = self.get_origin(headers)
message = f"""
You are invited by {sender} to join their archive, "{archive_name}" on Browsertrix Cloud!
You can join by clicking here: {self.host}/join/{token}?email={receiver_email}
You can join by clicking here: {origin}/join/{token}?email={receiver_email}
The invite token is: {token}"""
self._send_encrypted(receiver_email, message)
def send_existing_user_invite(self, receiver_email, sender, archive_name, token):
# pylint: disable=too-many-arguments
def send_existing_user_invite(
self, receiver_email, sender, archive_name, token, headers=None
):
"""Send email to invite new user"""
origin = self.get_origin(headers)
message = f"""
You are invited by {sender} to join their archive, "{archive_name}" on Browsertrix Cloud!
You can join by clicking here: {self.host}/invite/accept/{token}?email={receiver_email}
You can join by clicking here: {origin}/invite/accept/{token}?email={receiver_email}
The invite token is: {token}"""
self._send_encrypted(receiver_email, message)
def send_user_forgot_password(self, receiver_email, token):
def send_user_forgot_password(self, receiver_email, token, headers=None):
"""Send password reset email with token"""
origin = self.get_origin(headers)
message = f"""
We received your password reset request. Please click here: {self.host}/reset-password?token={token}
We received your password reset request. Please click here: {origin}/reset-password?token={token}
to create a new password
"""

11
backend/invites.py
View File

@ -64,6 +64,7 @@ class InviteOps:
new_user_invite: NewUserInvite,
inviter_email: str,
archive_name: Optional[str],
headers: Optional[dict],
):
"""Add invite for new user"""
@ -76,7 +77,11 @@ class InviteOps:
await self.invites.insert_one(new_user_invite.to_dict())
self.email.send_new_user_invite(
new_user_invite.email, inviter_email, archive_name, new_user_invite.id
new_user_invite.email,
inviter_email,
archive_name,
new_user_invite.id,
headers,
)
async def get_valid_invite(self, invite_token: str, user):
@ -115,6 +120,7 @@ class InviteOps:
user_manager,
archive=None,
allow_existing=False,
headers: dict = None,
):
"""create new invite for user to join, optionally an archive.
if allow_existing is false, don't allow invites to existing users"""
@ -141,6 +147,7 @@ class InviteOps:
),
user.email,
archive_name,
headers,
)
return True
@ -160,7 +167,7 @@ class InviteOps:
await user_manager.user_db.update(other_user)
self.email.send_existing_user_invite(
other_user.email, user.name, archive_name,invite_code
other_user.email, user.name, archive_name, invite_code, headers
)
return False

2
backend/main.py
View File

@ -85,7 +85,7 @@ def main():
app.include_router(archive_ops.router)
@app.get("/api/settings")
@app.get("/settings")
async def get_settings():
return settings

1
backend/requirements.txt
View File

@ -1,4 +1,5 @@
uvicorn
fastapi==0.70.0
fastapi-users[mongodb]==8.1.2
loguru
aiofiles

18
backend/users.py
View File

@ -163,7 +163,9 @@ class UserManager(BaseUserManager[UserCreate, UserDB]):
):
"""callback after password forgot"""
print(f"User {user.id} has forgot their password. Reset token: {token}")
self.email.send_user_forgot_password(user.email, token)
self.email.send_user_forgot_password(
user.email, token, request and request.headers
)
###pylint: disable=no-self-use, unused-argument
async def on_after_request_verify(
@ -171,7 +173,7 @@ class UserManager(BaseUserManager[UserCreate, UserDB]):
):
"""callback after verification request"""
self.email.send_user_validation(user.email, token)
self.email.send_user_validation(user.email, token, request and request.headers)
# ============================================================================
@ -240,13 +242,19 @@ def init_users_api(app, user_manager):
@users_router.post("/invite", tags=["invites"])
async def invite_user(
invite: InviteRequest,
request: Request,
user: User = Depends(current_active_user),
):
# if not user.is_superuser:
# raise HTTPException(status_code=403, detail="Not Allowed")
if not user.is_superuser:
raise HTTPException(status_code=403, detail="Not Allowed")
await user_manager.invites.invite_user(
invite, user, user_manager, archive=None, allow_existing=False
invite,
user,
user_manager,
archive=None,
allow_existing=False,
headers=request.headers,
)
return {"invited": "new_user"}

73
chart/nginx.conf
View File

@ -1,73 +0,0 @@
worker_processes 1;
error_log stderr;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /dev/stdout;
sendfile on;
keepalive_timeout 65;
include ./resolvers/resolvers.conf;
server {
listen 80 default_server;
server_name _;
proxy_buffering off;
proxy_buffers 16 64k;
proxy_buffer_size 64k;
root /usr/share/nginx/html;
index index.html index.htm;
error_page 500 501 502 503 504 /50x.html;
merge_slashes off;
location = /50x.html {
root /usr/share/nginx/html;
}
location ~* /watch/([^/]+)/([^/]+)/ws {
set $archive $1;
set $crawl $2;
#auth_request /authcheck;
proxy_pass http://$2.crawlers.svc.cluster.local:9037/ws;
proxy_set_header Host "localhost";
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
}
location ~* /watch/([^/]+)/([^/]+)/ {
set $archive $1;
set $crawl $2;
#auth_request /authcheck;
proxy_pass http://$2.crawlers.svc.cluster.local:9037/;
proxy_set_header Host "localhost";
}
location = /authcheck {
internal;
proxy_pass http://localhost:8000/archives/$archive/crawls/$crawl;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
}
location /healthz {
return 200;
}
location / {
#proxy_pass http://localhost:8000/;
#proxy_set_header Host $host;
root /usr/share/nginx/html;
index index.html index.htm;
}
}
}

9
chart/templates/frontend.yaml
View File

@ -48,6 +48,13 @@ spec:
mountPath: /etc/nginx/resolvers/
readOnly: true
env:
- name: BACKEND_HOST
value: {{ .Values.name }}-frontend
- name: BROWSER_SCREENCAST_URL
value: http://$2.crawlers.svc.cluster.local:9037
resources:
limits:
cpu: {{ .Values.nginx_limit_cpu }}
@ -57,7 +64,7 @@ spec:
readinessProbe:
httpGet:
path: /healthz
path: /
port: 80
---

15
docker-compose.yml
View File

@ -4,9 +4,6 @@ services:
backend:
build: ./backend
image: registry.digitalocean.com/btrix/webrecorder/browsertrix-api
ports:
- 8000:8000
volumes:
- /var/run/docker.sock:/var/run/docker.sock
@ -17,11 +14,21 @@ services:
- minio
- mongo
ports:
- 8000:8000
frontend:
build: ./frontend
image: registry.digitalocean.com/btrix/webrecorder/browsertrix-frontend
ports:
- 8010:80
- 9870:80
depends_on:
- backend
environment:
- BACKEND_HOST=backend
- BROWSER_SCREENCAST_URL=http://$$2:9037
redis:
image: redis

19
frontend/Dockerfile
View File

@ -1,5 +1,20 @@
FROM node:16 as build
WORKDIR /app
COPY . .
# disabled to speed up build, assuming built locally
#RUN yarn
RUN yarn build
FROM nginx
COPY ./dist/ /usr/share/nginx/html
COPY ./nginx.conf /etc/nginx/nginx.conf
COPY --from=build /app/dist /usr/share/nginx/html
COPY --from=build /app/nginx.conf.template /etc/nginx/templates/
#COPY ./dist /usr/share/nginx/html
#COPY ./nginx.conf.template /etc/nginx/templates/
RUN rm /etc/nginx/conf.d/*
RUN mkdir -p /etc/nginx/resolvers; echo "" > /etc/nginx/resolvers/resolvers.conf

78
frontend/nginx.conf
View File

@ -1,78 +0,0 @@
worker_processes 1;
error_log stderr;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /dev/stdout;
sendfile on;
keepalive_timeout 65;
include ./resolvers/resolvers.conf;
server {
listen 80 default_server;
server_name _;
proxy_buffering off;
proxy_buffers 16 64k;
proxy_buffer_size 64k;
root /usr/share/nginx/html;
index index.html index.htm;
error_page 500 501 502 503 504 /50x.html;
merge_slashes off;
location = /50x.html {
root /usr/share/nginx/html;
}
# fallback to index for any page
error_page 404 /index.html;
location ~* /watch/([^/]+)/([^/]+)/ws {
set $archive $1;
set $crawl $2;
#auth_request /authcheck;
proxy_pass http://$2.crawlers.svc.cluster.local:9037/ws;
proxy_set_header Host "localhost";
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
}
location ~* /watch/([^/]+)/([^/]+)/ {
set $archive $1;
set $crawl $2;
#auth_request /authcheck;
proxy_pass http://$2.crawlers.svc.cluster.local:9037/;
proxy_set_header Host "localhost";
}
location = /authcheck {
internal;
proxy_pass http://localhost:8000/archives/$archive/crawls/$crawl;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
}
location /healthz {
return 200;
}
location / {
#proxy_pass http://localhost:8000/;
#proxy_set_header Host $host;
root /usr/share/nginx/html;
index index.html index.htm;
}
}
}

63
frontend/nginx.conf.template Normal file
View File

@ -0,0 +1,63 @@
include ./resolvers/resolvers.conf;
server {
listen 80 default_server;
server_name _;
proxy_buffering off;
proxy_buffers 16 64k;
proxy_buffer_size 64k;
root /usr/share/nginx/html;
index index.html index.htm;
error_page 500 501 502 503 504 /50x.html;
merge_slashes off;
location = /50x.html {
root /usr/share/nginx/html;
}
# fallback to index for any page
error_page 404 /index.html;
location ~* /watch/([^/]+)/([^/]+)/ws {
set $archive $1;
set $crawl $2;
#auth_request /authcheck;
proxy_pass ${BROWSER_SCREENCAST_URL}/ws;
proxy_set_header Host "localhost";
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
}
location ~* /watch/([^/]+)/([^/]+)/ {
set $archive $1;
set $crawl $2;
#auth_request /authcheck;
proxy_pass ${BROWSER_SCREENCAST_URL}/;
proxy_set_header Host "localhost";
}
location = /authcheck {
internal;
proxy_pass http://localhost:8000/archives/$archive/crawls/$crawl;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
}
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
# used by docker only: k8s deployment handles /api directly via ingress
location /api/ {
proxy_pass http://${BACKEND_HOST}:8000/;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
}
}