profile browser fixes: better resource usage + load retry (main) (#1604)

- Backend: Use separate resource constraints for profiles: default
profile browser resources to either 'profile_browser_cpu' /
'profile_browser_memory' or single browser 'crawler_memory_base' /
'crawler_cpu_base', instead of scaled to the number of browser workers

- Frontend: check that profile html page is loading, keep retrying if
still getting nginx error instead of loading an iframe with the error.

Fixes #1598 (Copy of #1599 from 1.9.4)

backend/btrixcloud/operator/baseoperator.py

@@ -36,11 +36,13 @@ class K8sOpAPI(K8sAPI):
 
         self.has_pod_metrics = False
         self.compute_crawler_resources()
+        self.compute_profile_resources()
 
     def compute_crawler_resources(self):
         """compute memory / cpu resources for crawlers"""
         p = self.shared_params
         num = max(int(p["crawler_browser_instances"]) - 1, 0)
+        print("crawler resources")
         if not p.get("crawler_cpu"):
             base = parse_quantity(p["crawler_cpu_base"])
             extra = parse_quantity(p["crawler_extra_cpu_per_browser"])
@@ -63,6 +65,23 @@ class K8sOpAPI(K8sAPI):
         else:
             print(f"memory = {p['crawler_memory']}")
 
+    def compute_profile_resources(self):
+        """compute memory /cpu resources for a single profile browser"""
+        p = self.shared_params
+        # if no profile specific options provided, default to crawler base for one browser
+        profile_cpu = parse_quantity(
+            p.get("profile_browser_cpu") or p["crawler_cpu_base"]
+        )
+        profile_memory = parse_quantity(
+            p.get("profile_browser_memory") or p["crawler_memory_base"]
+        )
+        p["profile_cpu"] = profile_cpu
+        p["profile_memory"] = profile_memory
+
+        print("profile browser resources")
+        print(f"cpu = {profile_cpu}")
+        print(f"memory = {profile_memory}")
+
     async def async_init(self):
         """perform any async init here"""
         self.has_pod_metrics = await self.is_pod_metrics_available()

chart/app-templates/profilebrowser.yaml

@@ -78,8 +78,8 @@ spec:
 
       resources:
         limits:
-          memory: "{{ crawler_memory }}"
+          memory: "{{ profile_memory }}"
 
         requests:
-          cpu: "{{ crawler_cpu }}"
-          memory: "{{ crawler_memory }}"
+          cpu: "{{ profile_cpu }}"
+          memory: "{{ profile_memory }}"

chart/values.yaml

@@ -220,6 +220,11 @@ crawler_extra_memory_per_browser: 768Mi
 # crawler_memory = crawler_memory_base + crawler_memory_per_extra_browser * (crawler_browser_instances - 1)
 # crawler_memory:
 
+# optional: defaults to crawler_memory_base and crawler_cpu_base if not set
+# profile_browser_memory:
+#
+# profile_browser_cpu:
+
 # Other Crawler Settings
 # ----------------------
 

frontend/src/features/browser-profiles/profile-browser.ts

@@ -301,6 +301,18 @@ export class ProfileBrowser extends LiteElement {
 
       return;
     } else if (result.url) {
+      // check that the browser is actually available
+      // if not, continue waiting
+      // (will not work with local frontend due to CORS)
+      try {
+        const resp = await fetch(result.url, { method: "HEAD" });
+        if (!resp.ok) {
+          return;
+        }
+      } catch (e) {
+        // ignore
+      }
+
       if (this.initialNavigateUrl) {
         await this.navigateBrowser({ url: this.initialNavigateUrl });
       }