From eaf805506334589f676f3d9fedbf3befb1154d81 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sun, 5 Dec 2021 13:02:26 -0800 Subject: [PATCH] Support unified docker + k8s deployment (#58) - adapt nginx config to work both in docker and k8s, using env vars to set urls backend: additional fixes: - use env vars with nginx config - fix settings api route - when sending e-mail, use the Host header for verification urls when available - prepare Dockerfile with full build from scratch in image, (disabled 'yarn install' for faster builds for now) - fix accept invite api for existing user to /archives/accept-invite/{token} --- backend/archives.py | 12 ++++-- backend/emailsender.py | 46 ++++++++++++++++----- backend/invites.py | 11 ++++- backend/main.py | 2 +- backend/requirements.txt | 1 + backend/users.py | 18 +++++--- chart/nginx.conf | 73 -------------------------------- chart/templates/frontend.yaml | 9 +++- docker-compose.yml | 15 +++++-- frontend/Dockerfile | 19 ++++++++- frontend/nginx.conf | 78 ----------------------------------- frontend/nginx.conf.template | 63 ++++++++++++++++++++++++++++ 12 files changed, 167 insertions(+), 180 deletions(-) delete mode 100644 chart/nginx.conf delete mode 100644 frontend/nginx.conf create mode 100644 frontend/nginx.conf.template diff --git a/backend/archives.py b/backend/archives.py index 322beab5..37560c95 100644 --- a/backend/archives.py +++ b/backend/archives.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import Dict, Union, Literal, Optional from pydantic import BaseModel -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Request from db import BaseMongoModel @@ -298,18 +298,24 @@ def init_archives_api(app, mdb, user_manager, invites, user_dep: User): @router.post("/invite", tags=["invites"]) async def invite_user_to_archive( invite: InviteToArchiveRequest, + request: Request, archive: Archive = Depends(archive_owner_dep), user: User = Depends(user_dep), ): if await invites.invite_user( - invite, user, user_manager, archive=archive, allow_existing=True + invite, + user, + user_manager, + archive=archive, + allow_existing=True, + headers=request.headers, ): return {"invited": "new_user"} return {"invited": "existing_user"} - @app.get("/invite/accept/{token}", tags=["invites"]) + @app.post("/archives/invite-accept/{token}", tags=["invites"]) async def accept_invite(token: str, user: User = Depends(user_dep)): invite = invites.accept_user_invite(user, token) diff --git a/backend/emailsender.py b/backend/emailsender.py index 00af7878..fc9a7be2 100644 --- a/backend/emailsender.py +++ b/backend/emailsender.py @@ -14,7 +14,7 @@ class EmailSender: self.password = os.environ.get("EMAIL_PASSWORD") self.smtp_server = os.environ.get("EMAIL_SMTP_HOST") - self.host = os.environ.get("APP_ORIGIN") + self.default_origin = os.environ.get("APP_ORIGIN") def _send_encrypted(self, receiver, message): """Send Encrypted SMTP Message""" @@ -32,47 +32,71 @@ class EmailSender: server.login(self.sender, self.password) server.sendmail(self.sender, receiver, message) - def send_user_validation(self, receiver_email, token): + def get_origin(self, headers): + """ Return origin of the received request""" + scheme = headers.get("X-Forwarded-Proto") + if not scheme: + scheme = "http" + + host = headers.get("Host") + if not host: + host = self.default_origin + + return scheme + "://" + host + + def send_user_validation(self, receiver_email, token, headers=None): """Send email to validate registration email address""" + + origin = self.get_origin(headers) + message = f""" Please verify your registration for Browsertrix Cloud for {receiver_email} -You can verify by clicking here: {self.host}/verify?token={token} +You can verify by clicking here: {origin}/verify?token={token} The verification token is: {token}""" self._send_encrypted(receiver_email, message) - def send_new_user_invite(self, receiver_email, sender, archive_name, token): + # pylint: disable=too-many-arguments + def send_new_user_invite( + self, receiver_email, sender, archive_name, token, headers=None + ): """Send email to invite new user""" + origin = self.get_origin(headers) + message = f""" You are invited by {sender} to join their archive, "{archive_name}" on Browsertrix Cloud! -You can join by clicking here: {self.host}/join/{token}?email={receiver_email} +You can join by clicking here: {origin}/join/{token}?email={receiver_email} The invite token is: {token}""" self._send_encrypted(receiver_email, message) - def send_existing_user_invite(self, receiver_email, sender, archive_name, token): + # pylint: disable=too-many-arguments + def send_existing_user_invite( + self, receiver_email, sender, archive_name, token, headers=None + ): """Send email to invite new user""" + origin = self.get_origin(headers) message = f""" You are invited by {sender} to join their archive, "{archive_name}" on Browsertrix Cloud! -You can join by clicking here: {self.host}/invite/accept/{token}?email={receiver_email} +You can join by clicking here: {origin}/invite/accept/{token}?email={receiver_email} The invite token is: {token}""" self._send_encrypted(receiver_email, message) - - - def send_user_forgot_password(self, receiver_email, token): + def send_user_forgot_password(self, receiver_email, token, headers=None): """Send password reset email with token""" + origin = self.get_origin(headers) + message = f""" -We received your password reset request. Please click here: {self.host}/reset-password?token={token} +We received your password reset request. Please click here: {origin}/reset-password?token={token} to create a new password """ diff --git a/backend/invites.py b/backend/invites.py index 1743b856..0a62ed2a 100644 --- a/backend/invites.py +++ b/backend/invites.py @@ -64,6 +64,7 @@ class InviteOps: new_user_invite: NewUserInvite, inviter_email: str, archive_name: Optional[str], + headers: Optional[dict], ): """Add invite for new user""" @@ -76,7 +77,11 @@ class InviteOps: await self.invites.insert_one(new_user_invite.to_dict()) self.email.send_new_user_invite( - new_user_invite.email, inviter_email, archive_name, new_user_invite.id + new_user_invite.email, + inviter_email, + archive_name, + new_user_invite.id, + headers, ) async def get_valid_invite(self, invite_token: str, user): @@ -115,6 +120,7 @@ class InviteOps: user_manager, archive=None, allow_existing=False, + headers: dict = None, ): """create new invite for user to join, optionally an archive. if allow_existing is false, don't allow invites to existing users""" @@ -141,6 +147,7 @@ class InviteOps: ), user.email, archive_name, + headers, ) return True @@ -160,7 +167,7 @@ class InviteOps: await user_manager.user_db.update(other_user) self.email.send_existing_user_invite( - other_user.email, user.name, archive_name,invite_code + other_user.email, user.name, archive_name, invite_code, headers ) return False diff --git a/backend/main.py b/backend/main.py index 87c2c2d1..4515cc1f 100644 --- a/backend/main.py +++ b/backend/main.py @@ -85,7 +85,7 @@ def main(): app.include_router(archive_ops.router) - @app.get("/api/settings") + @app.get("/settings") async def get_settings(): return settings diff --git a/backend/requirements.txt b/backend/requirements.txt index 8367914a..384134e7 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,4 +1,5 @@ uvicorn +fastapi==0.70.0 fastapi-users[mongodb]==8.1.2 loguru aiofiles diff --git a/backend/users.py b/backend/users.py index 9b7c79f0..854629f7 100644 --- a/backend/users.py +++ b/backend/users.py @@ -163,7 +163,9 @@ class UserManager(BaseUserManager[UserCreate, UserDB]): ): """callback after password forgot""" print(f"User {user.id} has forgot their password. Reset token: {token}") - self.email.send_user_forgot_password(user.email, token) + self.email.send_user_forgot_password( + user.email, token, request and request.headers + ) ###pylint: disable=no-self-use, unused-argument async def on_after_request_verify( @@ -171,7 +173,7 @@ class UserManager(BaseUserManager[UserCreate, UserDB]): ): """callback after verification request""" - self.email.send_user_validation(user.email, token) + self.email.send_user_validation(user.email, token, request and request.headers) # ============================================================================ @@ -240,13 +242,19 @@ def init_users_api(app, user_manager): @users_router.post("/invite", tags=["invites"]) async def invite_user( invite: InviteRequest, + request: Request, user: User = Depends(current_active_user), ): - # if not user.is_superuser: - # raise HTTPException(status_code=403, detail="Not Allowed") + if not user.is_superuser: + raise HTTPException(status_code=403, detail="Not Allowed") await user_manager.invites.invite_user( - invite, user, user_manager, archive=None, allow_existing=False + invite, + user, + user_manager, + archive=None, + allow_existing=False, + headers=request.headers, ) return {"invited": "new_user"} diff --git a/chart/nginx.conf b/chart/nginx.conf deleted file mode 100644 index 5c745700..00000000 --- a/chart/nginx.conf +++ /dev/null @@ -1,73 +0,0 @@ -worker_processes 1; -error_log stderr; -pid /var/run/nginx.pid; -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /dev/stdout; - sendfile on; - keepalive_timeout 65; - include ./resolvers/resolvers.conf; - server { - listen 80 default_server; - server_name _; - proxy_buffering off; - proxy_buffers 16 64k; - proxy_buffer_size 64k; - root /usr/share/nginx/html; - index index.html index.htm; - error_page 500 501 502 503 504 /50x.html; - merge_slashes off; - location = /50x.html { - root /usr/share/nginx/html; - } - - location ~* /watch/([^/]+)/([^/]+)/ws { - set $archive $1; - set $crawl $2; - #auth_request /authcheck; - - proxy_pass http://$2.crawlers.svc.cluster.local:9037/ws; - proxy_set_header Host "localhost"; - - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $http_connection; - } - - location ~* /watch/([^/]+)/([^/]+)/ { - set $archive $1; - set $crawl $2; - #auth_request /authcheck; - - proxy_pass http://$2.crawlers.svc.cluster.local:9037/; - proxy_set_header Host "localhost"; - } - - location = /authcheck { - internal; - proxy_pass http://localhost:8000/archives/$archive/crawls/$crawl; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - } - - location /healthz { - return 200; - } - - location / { - #proxy_pass http://localhost:8000/; - #proxy_set_header Host $host; - root /usr/share/nginx/html; - index index.html index.htm; - } - } -} - diff --git a/chart/templates/frontend.yaml b/chart/templates/frontend.yaml index b02ff28f..fbb91226 100644 --- a/chart/templates/frontend.yaml +++ b/chart/templates/frontend.yaml @@ -48,6 +48,13 @@ spec: mountPath: /etc/nginx/resolvers/ readOnly: true + env: + - name: BACKEND_HOST + value: {{ .Values.name }}-frontend + + - name: BROWSER_SCREENCAST_URL + value: http://$2.crawlers.svc.cluster.local:9037 + resources: limits: cpu: {{ .Values.nginx_limit_cpu }} @@ -57,7 +64,7 @@ spec: readinessProbe: httpGet: - path: /healthz + path: / port: 80 --- diff --git a/docker-compose.yml b/docker-compose.yml index eddf729c..9ee620fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,6 @@ services: backend: build: ./backend image: registry.digitalocean.com/btrix/webrecorder/browsertrix-api - ports: - - 8000:8000 - volumes: - /var/run/docker.sock:/var/run/docker.sock @@ -17,11 +14,21 @@ services: - minio - mongo + ports: + - 8000:8000 + frontend: build: ./frontend image: registry.digitalocean.com/btrix/webrecorder/browsertrix-frontend ports: - - 8010:80 + - 9870:80 + + depends_on: + - backend + + environment: + - BACKEND_HOST=backend + - BROWSER_SCREENCAST_URL=http://$$2:9037 redis: image: redis diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 6dc05282..6c2df9ae 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,5 +1,20 @@ +FROM node:16 as build + +WORKDIR /app +COPY . . +# disabled to speed up build, assuming built locally +#RUN yarn +RUN yarn build + FROM nginx -COPY ./dist/ /usr/share/nginx/html -COPY ./nginx.conf /etc/nginx/nginx.conf +COPY --from=build /app/dist /usr/share/nginx/html +COPY --from=build /app/nginx.conf.template /etc/nginx/templates/ + +#COPY ./dist /usr/share/nginx/html +#COPY ./nginx.conf.template /etc/nginx/templates/ + +RUN rm /etc/nginx/conf.d/* + +RUN mkdir -p /etc/nginx/resolvers; echo "" > /etc/nginx/resolvers/resolvers.conf diff --git a/frontend/nginx.conf b/frontend/nginx.conf deleted file mode 100644 index 5df7220f..00000000 --- a/frontend/nginx.conf +++ /dev/null @@ -1,78 +0,0 @@ -worker_processes 1; -error_log stderr; -pid /var/run/nginx.pid; -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /dev/stdout; - sendfile on; - keepalive_timeout 65; - include ./resolvers/resolvers.conf; - server { - listen 80 default_server; - server_name _; - proxy_buffering off; - proxy_buffers 16 64k; - proxy_buffer_size 64k; - root /usr/share/nginx/html; - index index.html index.htm; - - error_page 500 501 502 503 504 /50x.html; - - merge_slashes off; - location = /50x.html { - root /usr/share/nginx/html; - } - - # fallback to index for any page - error_page 404 /index.html; - - location ~* /watch/([^/]+)/([^/]+)/ws { - set $archive $1; - set $crawl $2; - #auth_request /authcheck; - - proxy_pass http://$2.crawlers.svc.cluster.local:9037/ws; - proxy_set_header Host "localhost"; - - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $http_connection; - } - - location ~* /watch/([^/]+)/([^/]+)/ { - set $archive $1; - set $crawl $2; - #auth_request /authcheck; - - proxy_pass http://$2.crawlers.svc.cluster.local:9037/; - proxy_set_header Host "localhost"; - } - - location = /authcheck { - internal; - proxy_pass http://localhost:8000/archives/$archive/crawls/$crawl; - proxy_pass_request_body off; - proxy_set_header Content-Length ""; - } - - location /healthz { - return 200; - } - - location / { - #proxy_pass http://localhost:8000/; - #proxy_set_header Host $host; - root /usr/share/nginx/html; - index index.html index.htm; - } - } -} - diff --git a/frontend/nginx.conf.template b/frontend/nginx.conf.template new file mode 100644 index 00000000..2c9abd60 --- /dev/null +++ b/frontend/nginx.conf.template @@ -0,0 +1,63 @@ +include ./resolvers/resolvers.conf; + +server { + listen 80 default_server; + server_name _; + proxy_buffering off; + proxy_buffers 16 64k; + proxy_buffer_size 64k; + root /usr/share/nginx/html; + index index.html index.htm; + + error_page 500 501 502 503 504 /50x.html; + + merge_slashes off; + location = /50x.html { + root /usr/share/nginx/html; + } + + # fallback to index for any page + error_page 404 /index.html; + + location ~* /watch/([^/]+)/([^/]+)/ws { + set $archive $1; + set $crawl $2; + #auth_request /authcheck; + + proxy_pass ${BROWSER_SCREENCAST_URL}/ws; + proxy_set_header Host "localhost"; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $http_connection; + } + + location ~* /watch/([^/]+)/([^/]+)/ { + set $archive $1; + set $crawl $2; + #auth_request /authcheck; + + proxy_pass ${BROWSER_SCREENCAST_URL}/; + proxy_set_header Host "localhost"; + } + + location = /authcheck { + internal; + proxy_pass http://localhost:8000/archives/$archive/crawls/$crawl; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + } + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + # used by docker only: k8s deployment handles /api directly via ingress + location /api/ { + proxy_pass http://${BACKEND_HOST}:8000/; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + } +} +