Backend lint check (#451)

- apply lint + format fixes to backend
- add ci for lint + format fixes for backend
- use fixed version of pydantic

.github/workflows/lint.yaml

@@ -0,0 +1,28 @@
+name: Lint Check
+on: [push, pull_request]
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          cd backend/
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -U black pylint
+
+      - name: Style Check
+        run: |
+          black --check backend/btrixcloud/
+
+      - name: Lint Check
+        run: |
+          pylint backend/btrixcloud/

backend/btrixcloud/archives.py

@@ -12,8 +12,12 @@ from .db import BaseMongoModel
 
 from .users import User
 
-from .invites import AddToArchiveRequest, InvitePending, InviteToArchiveRequest, UserRole
+from .invites import (
-
+    AddToArchiveRequest,
+    InvitePending,
+    InviteToArchiveRequest,
+    UserRole,
+)
 
 # crawl scale for constraint
 MAX_CRAWL_SCALE = 3
@@ -223,6 +227,8 @@ class ArchiveOps:
 # ============================================================================
 def init_archives_api(app, mdb, user_manager, invites, user_dep: User):
     """Init archives api router for /archives"""
+    # pylint: disable=too-many-locals
+
     ops = ArchiveOps(mdb, invites)
 
     async def archive_dep(aid: str, user: User = Depends(user_dep)):
@@ -336,7 +342,6 @@ def init_archives_api(app, mdb, user_manager, invites, user_dep: User):
     @router.post("/add-user", tags=["invites"])
     async def add_new_user_to_archive(
         invite: AddToArchiveRequest,
-        request: Request,
         archive: Archive = Depends(archive_owner_dep),
         user: User = Depends(user_dep),
     ):

backend/btrixcloud/users.py

@@ -183,6 +183,7 @@ class UserManager(BaseUserManager[UserCreate, UserDB]):
     async def create_non_super_user(
         self, email: str, password: str, name: str = "New user"
     ):
+        """create a regular user with given credentials"""
         if not email:
             print("No user defined", flush=True)
             return
@@ -206,7 +207,6 @@ class UserManager(BaseUserManager[UserCreate, UserDB]):
         except (DuplicateKeyError, UserAlreadyExists):
             print(f"User {email} already exists", flush=True)
 
-
     async def on_after_register_custom(
         self, user: UserDB, user_create: UserCreate, request: Optional[Request]
     ):

backend/requirements.txt

@@ -3,6 +3,7 @@ fastapi==0.71.0
 fastapi-users[mongodb]==9.2.2
 loguru
 aiofiles
+pydantic==1.8.2
 kubernetes-asyncio==22.6.5
 aiobotocore
 redis>=4.2.0rc1

backend/test/conftest.py

@@ -11,6 +11,7 @@ ADMIN_PW = "PASSW0RD!"
 VIEWER_USERNAME = "viewer@example.com"
 VIEWER_PW = "viewerPASSW0RD!"
 
+
 @pytest.fixture(scope="session")
 def admin_auth_headers():
     r = requests.post(
@@ -25,12 +26,14 @@ def admin_auth_headers():
     access_token = data.get("access_token")
     return {"Authorization": f"Bearer {access_token}"}
 
+
 @pytest.fixture(scope="session")
 def admin_aid(admin_auth_headers):
     r = requests.get(f"{API_PREFIX}/archives", headers=admin_auth_headers)
     data = r.json()
     return data["archives"][0]["id"]
 
+
 @pytest.fixture(scope="session")
 def admin_crawl_id(admin_auth_headers, admin_aid):
     # Start crawl.
@@ -57,6 +60,7 @@ def admin_crawl_id(admin_auth_headers, admin_aid):
             return crawl_id
         time.sleep(5)
 
+
 @pytest.fixture(scope="session")
 def viewer_auth_headers(admin_auth_headers, admin_aid):
     requests.post(

backend/test/test_login.py

@@ -7,7 +7,11 @@ def test_login_invalid():
     password = "invalid"
     r = requests.post(
         f"{API_PREFIX}/auth/jwt/login",
-        data={"username": ADMIN_USERNAME, "password": password, "grant_type": "password"},
+        data={
+            "username": ADMIN_USERNAME,
+            "password": password,
+            "grant_type": "password",
+        },
     )
     data = r.json()
 
@@ -18,7 +22,11 @@ def test_login_invalid():
 def test_login():
     r = requests.post(
         f"{API_PREFIX}/auth/jwt/login",
-        data={"username": ADMIN_USERNAME, "password": ADMIN_PW, "grant_type": "password"},
+        data={
+            "username": ADMIN_USERNAME,
+            "password": ADMIN_PW,
+            "grant_type": "password",
+        },
     )
     data = r.json()
 

backend/test/test_permissions.py

@@ -5,8 +5,7 @@ from .conftest import API_PREFIX
 
 def test_admin_get_archive_crawls(admin_auth_headers, admin_aid, admin_crawl_id):
     r = requests.get(
-        f"{API_PREFIX}/archives/{admin_aid}/crawls",
+        f"{API_PREFIX}/archives/{admin_aid}/crawls", headers=admin_auth_headers
-        headers=admin_auth_headers
     )
     data = r.json()
     assert len(data["crawls"]) > 0
@@ -16,8 +15,7 @@ def test_admin_get_archive_crawls(admin_auth_headers, admin_aid, admin_crawl_id)
 
 def test_viewer_get_archive_crawls(viewer_auth_headers, admin_aid, admin_crawl_id):
     r = requests.get(
-        f"{API_PREFIX}/archives/{admin_aid}/crawls",
+        f"{API_PREFIX}/archives/{admin_aid}/crawls", headers=viewer_auth_headers
-        headers=viewer_auth_headers
     )
     data = r.json()
     crawls = data["crawls"]
@@ -31,7 +29,7 @@ def test_viewer_get_archive_crawls(viewer_auth_headers, admin_aid, admin_crawl_i
 def test_viewer_get_crawl(viewer_auth_headers, admin_aid, admin_crawl_id):
     r = requests.get(
         f"{API_PREFIX}/archives/{admin_aid}/crawls/{admin_crawl_id}",
-        headers=viewer_auth_headers
+        headers=viewer_auth_headers,
     )
     data = r.json()
     assert data["id"] == admin_crawl_id
@@ -41,7 +39,7 @@ def test_viewer_get_crawl(viewer_auth_headers, admin_aid, admin_crawl_id):
 def test_viewer_get_crawl_replay(viewer_auth_headers, admin_aid, admin_crawl_id):
     r = requests.get(
         f"{API_PREFIX}/archives/{admin_aid}/crawls/{admin_crawl_id}/replay.json",
-        headers=viewer_auth_headers
+        headers=viewer_auth_headers,
     )
     data = r.json()
     assert data["id"] == admin_crawl_id

backend/test/test_run_crawl.py

@@ -80,6 +80,7 @@ def test_wait_for_complete(admin_auth_headers, admin_aid, admin_crawl_id):
     wacz_size = data["resources"][0]["size"]
     wacz_hash = data["resources"][0]["hash"]
 
+
 def test_crawl_info(admin_auth_headers, admin_aid, admin_crawl_id):
     r = requests.get(
         f"{API_PREFIX}/archives/{admin_aid}/crawls/{admin_crawl_id}",
@@ -88,6 +89,7 @@ def test_crawl_info(admin_auth_headers, admin_aid, admin_crawl_id):
     data = r.json()
     assert data["fileSize"] == wacz_size
 
+
 def test_download_wacz():
     r = requests.get(host_prefix + wacz_path)
     assert r.status_code == 200

backend/test/test_users.py

@@ -10,6 +10,7 @@ def test_create_super_user(admin_auth_headers):
     assert token != "None"
     assert len(token) > 4
 
+
 def test_create_non_super_user(viewer_auth_headers):
     assert viewer_auth_headers
     auth = viewer_auth_headers["Authorization"]