avoid exception if 'errors' key doesn't exist (#1301)

- avoid exception if 'errors' (or 'files' keys) don't exist (part of
#1297)
- ensure 'errors' list always set on output model for consistency,
defaulting to empty list
- fix tests for 'errors' being an empty empty list
follow-up to #1300 (merging 1.7.1 release into main)

.github/workflows/k3d-ci.yaml

@@ -26,7 +26,7 @@ jobs:
             -p "30870:30870@agent:0:direct"
             --agents 1
             --no-lb
-            --k3s-arg "--disable=traefik,servicelb@server:*"
+            --k3s-arg "--disable=traefik,servicelb,metrics-server@server:*"
 
       - name: Checkout
         uses: actions/checkout@v3
@@ -63,6 +63,8 @@ jobs:
 
       - name: Install Kubectl
         uses: azure/setup-kubectl@v3
+        with:
+            version: v1.28.2
 
       - name: Install Helm
         uses: azure/setup-helm@v3

backend/btrixcloud/basecrawls.py

@@ -116,8 +116,8 @@ class BaseCrawlOps:
                     res.get("collectionIds")
                 )
 
-        del res["files"]
+        res.pop("files", None)
-        del res["errors"]
+        res.pop("errors", None)
 
         crawl = cls_type.from_dict(res)
 

backend/btrixcloud/models.py

@@ -461,7 +461,7 @@ class CrawlOut(BaseMongoModel):
 
     tags: Optional[List[str]] = []
 
-    errors: Optional[List[str]]
+    errors: Optional[List[str]] = []
 
     collectionIds: Optional[List[UUID4]] = []
 

backend/test/test_run_crawl.py

@@ -138,7 +138,7 @@ def test_crawls_exclude_errors(admin_auth_headers, default_org_id, admin_crawl_i
     )
     assert r.status_code == 200
     data = r.json()
-    assert "errors" not in data or data.get("errors") is None
+    assert data.get("errors") == []
 
     # replay.json endpoint
     r = requests.get(
@@ -147,7 +147,7 @@ def test_crawls_exclude_errors(admin_auth_headers, default_org_id, admin_crawl_i
     )
     assert r.status_code == 200
     data = r.json()
-    assert "errors" not in data or data.get("errors") is None
+    assert data.get("errors") == []
 
     # List endpoint
     r = requests.get(
@@ -157,7 +157,7 @@ def test_crawls_exclude_errors(admin_auth_headers, default_org_id, admin_crawl_i
     assert r.status_code == 200
     crawls = r.json()["items"]
     for crawl in crawls:
-        assert "errors" not in crawl or crawl.get("errors") is None
+        assert data.get("errors") == []
 
 
 def test_crawls_exclude_full_seeds(admin_auth_headers, default_org_id, admin_crawl_id):

backend/test/test_uploads.py

@@ -178,7 +178,7 @@ def test_get_upload_replay_json(
     assert data["resources"][0]["path"]
     assert data["resources"][0]["size"]
     assert data["resources"][0]["hash"]
-    assert data["errors"] == None
+    assert data["errors"] == []
     assert "files" not in data
 
 
@@ -201,7 +201,7 @@ def test_get_upload_replay_json_admin(
     assert data["resources"][0]["path"]
     assert data["resources"][0]["size"]
     assert data["resources"][0]["hash"]
-    assert data["errors"] == None
+    assert data["errors"] == []
     assert "files" not in data
 
 
@@ -748,7 +748,7 @@ def test_get_upload_replay_json_from_all_crawls(admin_auth_headers, default_org_
     assert data["resources"][0]["path"]
     assert data["resources"][0]["size"]
     assert data["resources"][0]["hash"]
-    assert data["errors"] == None
+    assert data["errors"] == []
     assert "files" not in data
 
 
@@ -769,7 +769,7 @@ def test_get_upload_replay_json_admin_from_all_crawls(
     assert data["resources"][0]["path"]
     assert data["resources"][0]["size"]
     assert data["resources"][0]["hash"]
-    assert data["errors"] == None
+    assert data["errors"] == []
     assert "files" not in data