diff --git a/.github/workflows/k3d-ci.yaml b/.github/workflows/k3d-ci.yaml index bba893bc..0945bd69 100644 --- a/.github/workflows/k3d-ci.yaml +++ b/.github/workflows/k3d-ci.yaml @@ -68,7 +68,7 @@ jobs: - name: Start Cluster with Helm run: | - helm upgrade --install -f ./chart/values.yaml -f ./chart/examples/k3d-ci.yaml btrix ./chart/ + helm upgrade --install -f ./chart/values.yaml -f ./chart/test/test.yaml btrix ./chart/ - name: Install Python uses: actions/setup-python@v3 diff --git a/.github/workflows/k3d-nightly-ci.yaml b/.github/workflows/k3d-nightly-ci.yaml new file mode 100644 index 00000000..2543d15a --- /dev/null +++ b/.github/workflows/k3d-nightly-ci.yaml @@ -0,0 +1,83 @@ +name: Nightly tests (K3d) + +on: + schedule: + # Run daily at 8am UTC + - cron: '0 8 1 * *' + +jobs: + btrix-k3d-nightly-test: + runs-on: ubuntu-latest + steps: + - name: Create k3d Cluster + uses: AbsaOSS/k3d-action@v2 + with: + cluster-name: btrix-nightly + args: >- + -p "30870:30870@agent:0:direct" + --agents 1 + --no-lb + --k3s-arg "--no-deploy=traefik,servicelb,metrics-server@server:*" + + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + driver-opts: network=host + + - name: Build Backend + uses: docker/build-push-action@v3 + with: + context: backend + load: true + #outputs: type=tar,dest=backend.tar + tags: webrecorder/browsertrix-backend:latest + cache-from: type=gha,scope=backend + cache-to: type=gha,scope=backend,mode=max + + - name: Build Frontend + uses: docker/build-push-action@v3 + with: + context: frontend + load: true + #outputs: type=tar,dest=frontend.tar + tags: webrecorder/browsertrix-frontend:latest + cache-from: type=gha,scope=frontend + cache-to: type=gha,scope=frontend,mode=max + + - name: 'Import Images' + run: | + k3d image import webrecorder/browsertrix-backend:latest -m direct -c btrix-nightly --verbose + k3d image import webrecorder/browsertrix-frontend:latest -m direct -c btrix-nightly --verbose + + - name: Install Kubectl + uses: azure/setup-kubectl@v3 + + - name: Install Helm + uses: azure/setup-helm@v3 + with: + version: 3.10.2 + + - name: Start Cluster with Helm + run: | + helm upgrade --install -f ./chart/values.yaml -f ./chart/test/test.yaml --set invite_expire_seconds=10 btrix ./chart/ + + - name: Install Python + uses: actions/setup-python@v3 + with: + python-version: '3.9' + + - name: Install Python Libs + run: pip install pytest requests + + - name: Wait for all pods to be ready + run: kubectl wait --for=condition=ready pod --all --timeout=240s + + - name: Run Tests + run: pytest -vv ./backend/test_nightly/*.py + + - name: Print Backend Logs + if: ${{ failure() }} + run: kubectl logs svc/browsertrix-cloud-backend diff --git a/.github/workflows/microk8s-ci.yaml b/.github/workflows/microk8s-ci.yaml index 1d5ee84b..83febfc2 100644 --- a/.github/workflows/microk8s-ci.yaml +++ b/.github/workflows/microk8s-ci.yaml @@ -46,7 +46,7 @@ jobs: - name: Start Cluster with Helm run: | - sudo microk8s helm3 upgrade --install -f ./chart/values.yaml -f ./chart/examples/microk8s-ci.yaml btrix ./chart/ + sudo microk8s helm3 upgrade --install -f ./chart/values.yaml -f ./chart/test/test.yaml -f ./chart/test/microk8s-ci.yaml btrix ./chart/ - name: Install Python uses: actions/setup-python@v3 diff --git a/backend/btrixcloud/db.py b/backend/btrixcloud/db.py index 59c914b8..b91f72dd 100644 --- a/backend/btrixcloud/db.py +++ b/backend/btrixcloud/db.py @@ -56,6 +56,7 @@ async def update_and_prepare_db( crawl_config_ops, crawls_ops, coll_ops, + invite_ops, ): """Prepare database for application. @@ -68,7 +69,7 @@ async def update_and_prepare_db( """ if await run_db_migrations(mdb): await drop_indexes(mdb) - await create_indexes(org_ops, crawl_config_ops, crawls_ops, coll_ops) + await create_indexes(org_ops, crawl_config_ops, crawls_ops, coll_ops, invite_ops) await user_manager.create_super_user() await org_ops.create_default_org() print("Database updated and ready", flush=True) @@ -121,13 +122,14 @@ async def drop_indexes(mdb): # ============================================================================ -async def create_indexes(org_ops, crawl_config_ops, crawls_ops, coll_ops): +async def create_indexes(org_ops, crawl_config_ops, crawls_ops, coll_ops, invite_ops): """Create database indexes.""" print("Creating database indexes", flush=True) await org_ops.init_index() await crawl_config_ops.init_index() await crawls_ops.init_index() await coll_ops.init_index() + await invite_ops.init_index() # ============================================================================ diff --git a/backend/btrixcloud/invites.py b/backend/btrixcloud/invites.py index 25ed928b..c85b8075 100644 --- a/backend/btrixcloud/invites.py +++ b/backend/btrixcloud/invites.py @@ -5,8 +5,10 @@ from enum import IntEnum from typing import Optional import os import urllib.parse +import time import uuid +from pymongo.errors import AutoReconnect from pydantic import BaseModel, UUID4 from fastapi import HTTPException @@ -67,6 +69,25 @@ class InviteOps: self.email = email self.allow_dupe_invites = os.environ.get("ALLOW_DUPE_INVITES", "0") == "1" + async def init_index(self): + """Create TTL index so that invites auto-expire""" + while True: + try: + # Default to 7 days + expire_after_seconds = int( + os.environ.get("INVITE_EXPIRE_SECONDS", "604800") + ) + return await self.invites.create_index( + "created", expireAfterSeconds=expire_after_seconds + ) + # pylint: disable=duplicate-code + except AutoReconnect: + print( + "Database connection unavailable to create index. Will try again in 5 scconds", + flush=True, + ) + time.sleep(5) + async def add_new_user_invite( self, new_user_invite: InvitePending, diff --git a/backend/btrixcloud/main.py b/backend/btrixcloud/main.py index 2b7ebc85..d7ea1d98 100644 --- a/backend/btrixcloud/main.py +++ b/backend/btrixcloud/main.py @@ -108,7 +108,7 @@ def main(): asyncio.create_task( update_and_prepare_db( - mdb, user_manager, org_ops, crawl_config_ops, crawls, coll_ops + mdb, user_manager, org_ops, crawl_config_ops, crawls, coll_ops, invites ) ) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 5e98ce61..4a417561 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -167,6 +167,7 @@ class OrgOps: while True: try: return await self.orgs.create_index("name", unique=True) + # pylint: disable=duplicate-code except AutoReconnect: print( "Database connection unavailable to create index. Will try again in 5 scconds", diff --git a/backend/test/test_invites.py b/backend/test/test_invites.py index e4081701..ce58f279 100644 --- a/backend/test/test_invites.py +++ b/backend/test/test_invites.py @@ -1,4 +1,5 @@ import requests +import time import pytest diff --git a/backend/test_nightly/__init__.py b/backend/test_nightly/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/test_nightly/conftest.py b/backend/test_nightly/conftest.py new file mode 100644 index 00000000..d78b00d5 --- /dev/null +++ b/backend/test_nightly/conftest.py @@ -0,0 +1,43 @@ +import pytest +import requests +import time + + +HOST_PREFIX = "http://127.0.0.1:30870" +API_PREFIX = HOST_PREFIX + "/api" + +ADMIN_USERNAME = "admin@example.com" +ADMIN_PW = "PASSW0RD!" + + +@pytest.fixture(scope="session") +def admin_auth_headers(): + while True: + r = requests.post( + f"{API_PREFIX}/auth/jwt/login", + data={ + "username": ADMIN_USERNAME, + "password": ADMIN_PW, + "grant_type": "password", + }, + ) + data = r.json() + try: + return {"Authorization": f"Bearer {data['access_token']}"} + except: + print("Waiting for admin_auth_headers") + time.sleep(5) + + +@pytest.fixture(scope="session") +def default_org_id(admin_auth_headers): + while True: + r = requests.get(f"{API_PREFIX}/orgs", headers=admin_auth_headers) + data = r.json() + try: + for org in data["orgs"]: + if org["default"] is True: + return org["id"] + except: + print("Waiting for default org id") + time.sleep(5) diff --git a/backend/test_nightly/test_invite_expiration.py b/backend/test_nightly/test_invite_expiration.py new file mode 100644 index 00000000..50c1d21c --- /dev/null +++ b/backend/test_nightly/test_invite_expiration.py @@ -0,0 +1,44 @@ +import requests + +from .conftest import API_PREFIX + + +def test_invites_expire(admin_auth_headers, default_org_id): + # Send invite + INVITE_EMAIL = "invite-expires@example.com" + r = requests.post( + f"{API_PREFIX}/orgs/{default_org_id}/invite", + headers=admin_auth_headers, + json={"email": INVITE_EMAIL, "role": 10}, + ) + assert r.status_code == 200 + data = r.json() + assert data["invited"] == "new_user" + + # Verify invite exists + r = requests.get( + f"{API_PREFIX}/orgs/{default_org_id}/invites", + headers=admin_auth_headers, + ) + assert r.status_code == 200 + data = r.json() + invites_matching_email = [ + invite for invite in data["pending_invites"] if invite["email"] == INVITE_EMAIL + ] + assert len(invites_matching_email) == 1 + + # Wait two minutes to give Mongo sufficient time to delete the invite + # See: https://www.mongodb.com/docs/manual/core/index-ttl/#timing-of-the-delete-operation + time.sleep(120) + + # Check invites again and verify invite has been removed + r = requests.get( + f"{API_PREFIX}/orgs/{default_org_id}/invites", + headers=admin_auth_headers, + ) + assert r.status_code == 200 + data = r.json() + invites_matching_email = [ + invite for invite in data["pending_invites"] if invite["email"] == INVITE_EMAIL + ] + assert len(invites_matching_email) == 0 diff --git a/chart/examples/microk8s-ci.yaml b/chart/examples/microk8s-ci.yaml deleted file mode 100644 index ab65429c..00000000 --- a/chart/examples/microk8s-ci.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# microk8s overrides for ci -# ------------------------- - - -# use local images -backend_image: "localhost:32000/webrecorder/browsertrix-backend:latest" -frontend_image: "localhost:32000/webrecorder/browsertrix-frontend:latest" - -# don't pull use, existing images -backend_pull_policy: "IfNotPresent" -frontend_pull_policy: "IfNotPresent" - - -mongo_auth: - # specify either username + password (for local mongo) - username: root - password: PASSWORD@ - - -superuser: - # set this to enable a superuser admin - email: admin@example.com - - # optional: if not set, automatically generated - # change or remove this - password: PASSW0RD! - - -local_service_port: 30870 - diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index a7025ad3..a0ed3ae9 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -27,6 +27,8 @@ data: DEFAULT_ORG: "{{ .Values.default_org }}" + INVITE_EXPIRE_SECONDS: "{{ .Values.invite_expire_seconds }}" + JOB_IMAGE: "{{ .Values.backend_image }}" JOB_PULL_POLICY: "{{ .Values.backend_pull_policy }}" diff --git a/chart/test/microk8s-ci.yaml b/chart/test/microk8s-ci.yaml new file mode 100644 index 00000000..3b105def --- /dev/null +++ b/chart/test/microk8s-ci.yaml @@ -0,0 +1,6 @@ +# microk8s overrides for ci +# ------------------------- + +# use local images +backend_image: "localhost:32000/webrecorder/browsertrix-backend:latest" +frontend_image: "localhost:32000/webrecorder/browsertrix-frontend:latest" diff --git a/chart/examples/k3d-ci.yaml b/chart/test/test.yaml similarity index 81% rename from chart/examples/k3d-ci.yaml rename to chart/test/test.yaml index 12caf8f5..353850b5 100644 --- a/chart/examples/k3d-ci.yaml +++ b/chart/test/test.yaml @@ -1,12 +1,9 @@ -# k3s overrides for ci -# -------------------- +# test overrides +# -------------- - -# don't pull use, existing images backend_pull_policy: "Never" frontend_pull_policy: "Never" - mongo_auth: # specify either username + password (for local mongo) username: root @@ -23,4 +20,3 @@ superuser: local_service_port: 30870 - diff --git a/chart/values.yaml b/chart/values.yaml index 1aea9032..86b50ff8 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -24,6 +24,9 @@ allow_dupe_invites: "0" # number of workers for backend api backend_workers: 4 +# number of seconds before pending invites expire - default is 7 days +invite_expire_seconds: 604800 + # base url for replayweb.page rwp_base_url: "https://replayweb.page/" diff --git a/frontend/src/pages/join.ts b/frontend/src/pages/join.ts index 039ed1ba..8ae6f71d 100644 --- a/frontend/src/pages/join.ts +++ b/frontend/src/pages/join.ts @@ -100,6 +100,10 @@ export class Join extends LiteElement { inviterName: body.inviterName, orgName: body.orgName, }; + } else if (resp.status === 404) { + this.serverError = msg( + "This invite doesn't exist or has expired. Please ask the organization administrator to resend an invitation." + ); } else { this.serverError = msg("This invitation is not valid"); }