Add Waiting state on the backend and frontend (#839)

* operator: add waiting state
- add pods as related objects
- inspect pod status, set crawl status to 'waiting' if no pods are running

frontend:
- frontend support for 'waiting' state
- show waiting icon from mocks

---------
Co-authored-by: Henry Wilkinson <henry@wilkinson.graphics>

backend/btrixcloud/operator.py

@@ -32,6 +32,7 @@ from .crawls import (
 STS = "StatefulSet.apps/v1"
 CMAP = "ConfigMap.v1"
 PVC = "PersistentVolumeClaim.v1"
+POD = "Pod.v1"
 
 DEFAULT_TTL = 30
 
@@ -186,9 +187,10 @@ class BtrixOperator(K8sAPI):
         crawl_sts = f"crawl-{crawl_id}"
         redis_sts = f"redis-{crawl_id}"
 
-        has_crawl_children = STS in data.children and crawl_sts in data.children[STS]
+        has_crawl_children = crawl_sts in data.children[STS]
         if has_crawl_children:
-            status = await self.sync_crawl_state(redis_url, crawl, status)
+            pods = data.related[POD]
+            status = await self.sync_crawl_state(redis_url, crawl, status, pods)
         elif not status.finished:
             status.state = "starting"
 
@@ -220,7 +222,7 @@ class BtrixOperator(K8sAPI):
                 "spec"
             ]["volumeClaimTemplates"]
 
-        has_redis_children = STS in data.children and redis_sts in data.children[STS]
+        has_redis_children = redis_sts in data.children[STS]
         if has_redis_children:
             children[2]["spec"]["volumeClaimTemplates"] = data.children[STS][redis_sts][
                 "spec"
@@ -251,6 +253,13 @@ class BtrixOperator(K8sAPI):
                     "resource": "persistentvolumeclaims",
                     "labelSelector": {"matchLabels": {"crawl": crawl_id}},
                 },
+                {
+                    "apiVersion": "v1",
+                    "resource": "pods",
+                    "labelSelector": {
+                        "matchLabels": {"crawl": crawl_id, "role": "crawler"}
+                    },
+                },
             ]
         }
 
@@ -328,7 +337,7 @@ class BtrixOperator(K8sAPI):
         except:
             return None
 
-    async def sync_crawl_state(self, redis_url, crawl, status):
+    async def sync_crawl_state(self, redis_url, crawl, status, pods):
         """sync crawl state for running crawl"""
         redis = await self._get_redis(redis_url)
         if not redis:
@@ -354,7 +363,7 @@ class BtrixOperator(K8sAPI):
             status.filesAdded = int(await redis.get("filesAdded") or 0)
 
             # update stats and get status
-            return await self.update_crawl_state(redis, crawl, status)
+            return await self.update_crawl_state(redis, crawl, status, pods)
 
         # pylint: disable=broad-except
         except Exception as exc:
@@ -362,6 +371,20 @@ class BtrixOperator(K8sAPI):
             print(f"Crawl get failed: {exc}, will try again")
             return status
 
+    async def check_if_pods_running(self, pods):
+        """check if at least one crawler pod has started"""
+        try:
+            for pod in pods.values():
+                print("Phase", pod["status"]["phase"])
+                if pod["status"]["phase"] == "Running":
+                    return True
+        # pylint: disable=bare-except
+        except:
+            # assume no valid pod found
+            pass
+
+        return False
+
     async def add_file_to_crawl(self, cc_data, crawl):
         """Handle finished CrawlFile to db"""
 
@@ -386,7 +409,7 @@ class BtrixOperator(K8sAPI):
 
         return True
 
-    async def update_crawl_state(self, redis, crawl, status):
+    async def update_crawl_state(self, redis, crawl, status, pods):
         """update crawl state and check if crawl is now done"""
         results = await redis.hvals(f"{crawl.id}:status")
         stats = await get_redis_crawl_stats(redis, crawl.id)
@@ -405,6 +428,15 @@ class BtrixOperator(K8sAPI):
             # backwards compatibility with older crawler
             await redis.set("crawl-stop", "1")
 
+        # check if at least one pod started running
+        # otherwise, mark as 'waiting' and return
+        if not await self.check_if_pods_running(pods):
+            if status.state not in ("waiting", "canceled"):
+                await update_crawl(self.crawls, crawl.id, state="waiting")
+                status.state = "waiting"
+
+            return status
+
         # optimization: don't update db once crawl is already running
         # will set stats at when crawl is finished, otherwise can read
         # directly from redis

backend/test/test_run_crawl.py

@@ -61,7 +61,7 @@ def test_wait_for_complete(admin_auth_headers, default_org_id, admin_crawl_id):
 
     # ensure filename matches specified pattern
     # set in default_crawl_filename_template
-    assert re.search('/[\\d]+-testing-[\\w-]+\.wacz', data["resources"][0]["path"])
+    assert re.search("/[\\d]+-testing-[\\w-]+\\.wacz", data["resources"][0]["path"])
 
     assert data["tags"] == ["wr-test-1", "wr-test-2"]
 

frontend/src/components/crawl-status.ts

@@ -76,6 +76,17 @@ export class CrawlStatus extends LitElement {
         break;
       }
 
+      case "waiting": {
+        icon = html`<sl-icon
+          name="hourglass-split"
+          class="animatePulse"
+          slot="prefix"
+          style="color: var(--sl-color-purple-600)"
+        ></sl-icon>`;
+        label = msg("Waiting");
+        break;
+      }
+
       case "running": {
         icon = html`<sl-icon
           name="dot"

frontend/src/pages/org/crawl-detail.ts

@@ -90,6 +90,7 @@ export class CrawlDetail extends LiteElement {
     return (
       this.crawl.state === "running" ||
       this.crawl.state === "starting" ||
+      this.crawl.state === "waiting" ||
       this.crawl.state === "stopping"
     );
   }

frontend/src/pages/org/workflow-detail.ts

@@ -794,15 +794,16 @@ export class WorkflowDetail extends LiteElement {
     if (!this.authState || !this.workflow?.currCrawlState) return "";
 
     const isStarting = this.workflow.currCrawlState === "starting";
+    const isWaiting = this.workflow.currCrawlState === "waiting";
     const isRunning = this.workflow.currCrawlState === "running";
     const isStopping = this.workflow.currCrawlState === "stopping";
     const authToken = this.authState.headers.Authorization.split(" ")[1];
 
     return html`
-      ${isStarting
+      ${isStarting || isWaiting
         ? html`<div class="rounded border p-3">
             <p class="text-sm text-neutral-600 motion-safe:animate-pulse">
-              ${msg("Crawl starting...")}
+              ${isStarting ? msg("Crawl starting...") : msg("Crawl waiting for available resources before it can start...")}
             </p>
           </div>`
         : isActive(this.workflow.currCrawlState)

frontend/src/types/crawler.ts

@@ -90,6 +90,7 @@ export type Profile = {
 
 export type CrawlState =
   | "starting"
+  | "waiting"
   | "running"
   | "complete"
   | "failed"

frontend/src/utils/crawler.ts

@@ -1,6 +1,7 @@
 import type { CrawlState } from "../types/crawler";
 export const activeCrawlStates: CrawlState[] = [
   "starting",
+  "waiting",
   "running",
   "stopping",
 ];