1.10.2 release merge (#1866)

Merge 1.10.2 release into 1.11.0-beta.0

backend/btrixcloud/basecrawls.py

@@ -64,7 +64,8 @@ class BaseCrawlOps:
     background_job_ops: BackgroundJobOps
     page_ops: PageOps
 
-    presign_duration: int
+    presign_duration_seconds: int
+    expire_at_duration_seconds: int
 
     def __init__(
         self,
@@ -95,6 +96,9 @@ class BaseCrawlOps:
             min(presign_duration_minutes, PRESIGN_MINUTES_MAX) * 60
         )
 
+        # renew when <25% of time remaining
+        self.expire_at_duration_seconds = int(self.presign_duration_seconds * 0.75)
+
     def set_page_ops(self, page_ops):
         """set page ops reference"""
         self.page_ops = page_ops
@@ -434,7 +438,7 @@ class BaseCrawlOps:
             print("no files")
             return []
 
-        delta = timedelta(seconds=self.presign_duration_seconds)
+        delta = timedelta(seconds=self.expire_at_duration_seconds)
 
         out_files = []
 

backend/btrixcloud/operator/crawls.py

@@ -325,8 +325,15 @@ class CrawlOperator(BaseOperator):
         qa_source_crawl_id = params["qa_source_crawl_id"]
         name = f"qa-replay-{qa_source_crawl_id}"
 
-        if name in children[CMAP]:
+        configmap = children[CMAP].get(name)
-            return [children[CMAP][name]]
+        if configmap and not self._qa_configmap_update_needed(name, configmap):
+            metadata = configmap["metadata"]
+            configmap["metadata"] = {
+                "name": metadata["name"],
+                "namespace": metadata["namespace"],
+                "labels": metadata["labels"],
+            }
+            return [configmap]
 
         crawl_replay = await self.crawl_ops.get_internal_crawl_out(qa_source_crawl_id)
 
@@ -364,6 +371,22 @@ class CrawlOperator(BaseOperator):
 
         return self.load_from_yaml("crawler.yaml", params)
 
+    def _qa_configmap_update_needed(self, name, configmap):
+        try:
+            now = dt_now()
+            resources = json.loads(configmap["data"]["qa-config.json"])["resources"]
+            for resource in resources:
+                expire_at = datetime.fromisoformat(resource["expireAt"])
+                if expire_at <= now:
+                    print(f"Refreshing QA configmap for QA run: {name}")
+                    return True
+
+        # pylint: disable=broad-exception-caught
+        except Exception as e:
+            print(e)
+
+        return False
+
     # pylint: disable=too-many-arguments
     async def _resolve_scale(
         self,

chart/app-templates/crawler.yaml

@@ -124,7 +124,7 @@ spec:
         - {{ redis_url }}
       {% if qa_source_crawl_id %}
         - --qaSource
-        - /tmp/qa-config.json
+        - /tmp/qa/qa-config.json
       {% elif profile_filename %}
         - --profile
         - "@{{ profile_filename }}"
@@ -137,8 +137,7 @@ spec:
 
       {% if qa_source_crawl_id %}
         - name: qa-config
-          mountPath: /tmp/qa-config.json
+          mountPath: /tmp/qa/
-          subPath: qa-config.json
           readOnly: True
       {% endif %}
 

chart/templates/operators.yaml

@@ -23,7 +23,7 @@ spec:
   - apiVersion: v1
     resource: configmaps
     updateStrategy:
-      method: OnDelete
+      method: InPlace
 
   hooks:
     sync:

frontend/package.json

@@ -10,7 +10,7 @@
     "@lit/task": "^1.0.0",
     "@novnc/novnc": "^1.4.0-beta",
     "@rollup/plugin-commonjs": "^18.0.0",
-    "@shoelace-style/shoelace": "~2.15.0",
+    "@shoelace-style/shoelace": "~2.15.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@types/color": "^3.0.2",
     "@types/diff": "^5.0.9",

frontend/src/components/ui/config-details.ts

@@ -13,6 +13,7 @@ import type { CrawlConfig, Seed, SeedConfig } from "@/pages/org/types";
 import type { Collection } from "@/types/collection";
 import { isApiError } from "@/utils/api";
 import type { AuthState } from "@/utils/AuthService";
+import { DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler";
 import { humanizeSchedule } from "@/utils/cron";
 import LiteElement, { html } from "@/utils/LiteElement";
 
@@ -141,6 +142,9 @@ export class ConfigDetails extends LiteElement {
             `,
             () => this.renderSetting(msg("Exclusions"), msg("None")),
           )}
+          <btrix-section-heading style="--margin: var(--sl-spacing-medium)">
+            <h4>${msg("Per-Crawl Limits")}</h4>
+          </btrix-section-heading>
           ${this.renderSetting(
             msg("Max Pages"),
             when(
@@ -157,6 +161,21 @@ export class ConfigDetails extends LiteElement {
                   : undefined,
             ),
           )}
+          ${this.renderSetting(
+            msg("Crawl Time Limit"),
+            renderTimeLimit(crawlConfig?.crawlTimeout, Infinity),
+          )}
+          ${this.renderSetting(
+            msg("Crawl Size Limit"),
+            renderSize(crawlConfig?.maxCrawlSize),
+          )}
+          ${this.renderSetting(
+            msg("Crawler Instances"),
+            crawlConfig?.scale ? `${crawlConfig.scale}×` : "",
+          )}
+          <btrix-section-heading style="--margin: var(--sl-spacing-medium)">
+            <h4>${msg("Per-Page Limits")}</h4>
+          </btrix-section-heading>
           ${this.renderSetting(
             msg("Page Load Timeout"),
             renderTimeLimit(
@@ -169,7 +188,7 @@ export class ConfigDetails extends LiteElement {
             renderTimeLimit(crawlConfig?.config.postLoadDelay, 0),
           )}
           ${this.renderSetting(
-            msg("Page Behavior Timeout"),
+            msg("Behavior Timeout"),
             renderTimeLimit(
               crawlConfig?.config.behaviorTimeout,
               this.orgDefaults?.behaviorTimeoutSeconds ?? Infinity,
@@ -188,18 +207,6 @@ export class ConfigDetails extends LiteElement {
             msg("Delay Before Next Page"),
             renderTimeLimit(crawlConfig?.config.pageExtraDelay, 0),
           )}
-          ${this.renderSetting(
-            msg("Crawl Time Limit"),
-            renderTimeLimit(crawlConfig?.crawlTimeout, Infinity),
-          )}
-          ${this.renderSetting(
-            msg("Crawl Size Limit"),
-            renderSize(crawlConfig?.maxCrawlSize),
-          )}
-          ${this.renderSetting(
-            msg("Crawler Instances"),
-            crawlConfig?.scale ? `${crawlConfig.scale}×` : "",
-          )}
         </btrix-desc-list>
       </section>
       <section id="browser-settings" class="mb-8">
@@ -221,7 +228,11 @@ export class ConfigDetails extends LiteElement {
                 >
                   ${crawlConfig?.profileName}
                 </a>`,
-              () => crawlConfig?.profileName || msg("Default Profile"),
+              () =>
+                crawlConfig?.profileName ||
+                html`<span class="text-neutral-400"
+                  >${msg("Default Profile")}</span
+                >`,
             ),
           )}
           ${this.renderSetting(
@@ -237,7 +248,7 @@ export class ConfigDetails extends LiteElement {
             msg("User Agent"),
             crawlConfig?.config.userAgent
               ? crawlConfig.config.userAgent
-              : msg("Default User Agent"),
+              : html`<span class="text-neutral-400">${msg("Default")}</span>`,
           )}
           ${crawlConfig?.config.lang
             ? this.renderSetting(
@@ -396,15 +407,15 @@ export class ConfigDetails extends LiteElement {
         true,
       )}
       ${when(
-        ["host", "domain", "custom", "any"].includes(
+        DEPTH_SUPPORTED_SCOPES.includes(
           primarySeedConfig!.scopeType || seedsConfig.scopeType!,
         ),
         () =>
           this.renderSetting(
             msg("Max Depth"),
-            primarySeedConfig?.depth
+            primarySeedConfig && primarySeedConfig.depth !== null
               ? msg(str`${primarySeedConfig.depth} hop(s)`)
-              : msg("None"),
+              : msg("Unlimited (default)"),
           ),
       )}
       ${this.renderSetting(

frontend/src/features/archived-items/archived-item-list.ts

@@ -45,6 +45,14 @@ export class ArchivedItemListItem extends TailwindElement {
         var(--btrix-border-radius-bottom, 0);
       height: 2.5rem;
     }
+
+    sl-progress-ring {
+      /* Setting size to var(--font-size-base) breaks in chrome,
+      have cell contents inherit size from cell instead */
+      --size: 1em;
+      --track-width: 1px;
+      --indicator-width: 2px;
+    }
   `;
 
   @property({ type: Object, attribute: false })
@@ -141,7 +149,7 @@ export class ArchivedItemListItem extends TailwindElement {
               </btrix-table-cell>
             `
           : nothing}
-        <btrix-table-cell class="pr-0">
+        <btrix-table-cell class="pr-0 text-base">
           ${this.showStatus
             ? html`
                 <btrix-crawl-status
@@ -158,7 +166,7 @@ export class ArchivedItemListItem extends TailwindElement {
                   hoist
                 >
                   <sl-icon
-                    class="text-base"
+                    class="text-inherit"
                     style="color: ${crawlStatus.cssColor}"
                     name=${typeIcon}
                     label=${typeLabel}
@@ -179,12 +187,12 @@ export class ArchivedItemListItem extends TailwindElement {
               ? html`
                   <sl-progress-ring
                     value="${activeProgress}"
-                    style="color: ${qaStatus.cssColor}; --size: var(--font-size-base); --track-width: 1px; --indicator-width: 2px;"
+                    style="color: ${qaStatus.cssColor};"
                   ></sl-progress-ring>
                 `
               : html`
                   <sl-icon
-                    class="text-base"
+                    class="text-inherit"
                     style="color: ${qaStatus.cssColor}"
                     name=${isUpload ? "slash" : "microscope"}
                     library=${isUpload ? "default" : "app"}

frontend/src/index.ejs

@@ -8,7 +8,7 @@
     />
     <title>Browsertrix</title>
     <base href="/" />
-    <script defer src="/replay/ui.js"></script>
+
     <script
       src="https://browser.sentry-cdn.com/5.5.0/bundle.min.js"
       crossorigin="anonymous"
@@ -38,5 +38,14 @@
         });
       }
     </script>
+    <script>
+      customElements.whenDefined("browsertrix-app").then(() => {
+        // Load replay UI after browsertrix app is defined to prevent issues
+        // with re-defining shoelace components
+        const script = document.createElement("script");
+        script.src = "/replay/ui.js";
+        document.body.appendChild(script);
+      });
+    </script>
   </body>
 </html>

frontend/src/pages/org/archived-item-detail/ui/qa.ts

@@ -440,7 +440,6 @@ export class ArchivedItemDetailQA extends TailwindElement {
           <div slot="footer" class="flex justify-between">
             <sl-button
               size="small"
-              .autofocus=${true}
               @click=${() => void this.deleteQADialog?.hide()}
             >
               ${msg("Cancel")}

frontend/src/pages/org/archived-item-qa/archived-item-qa.ts

@@ -508,6 +508,7 @@ export class ArchivedItemQA extends TailwindElement {
 
         <div class="grid--tabGroup flex min-w-0 flex-col">
           <nav
+            aria-label="${msg("Page heuristics")}"
             class="-mx-3 my-0 flex gap-2 overflow-x-auto px-3 py-2 lg:mx-0 lg:px-0"
           >
             <btrix-navigation-button

frontend/src/pages/org/archived-item-qa/ui/text.ts

@@ -20,7 +20,7 @@ function renderDiff(
     diffImport.then(({ diffWords }) => {
       const diff = diffWords(crawlText, qaText);
 
-      const addedText = tw`bg-red-100 text-red-700`;
+      const addedText = tw`bg-red-100 text-red-700 no-underline`;
       const removedText = tw`bg-red-100 text-red-100`;
 
       return html`
@@ -29,16 +29,23 @@ function renderDiff(
           aria-labelledby="crawlTextHeading"
         >
           ${diff.map((part) => {
-            return html`
+            if (part.added) {
-              <span
+              return html`<del
-                class=${part.added
+                aria-label="${msg("Missing text: Crawl")}"
-                  ? removedText
+                class="${removedText}"
-                  : part.removed
+                >${part.value}</del
-                    ? addedText
+              >`;
-                    : ""}
+            } else if (part.removed) {
+              return html`<ins
+                aria-label="${msg("Added text: Crawl")}"
+                class="${addedText}"
+                >${part.value}</ins
+              >`;
+            } else {
+              return html`<span aria-label="${msg("Identical text")}"
                 >${part.value}</span
-              >
+              >`;
-            `;
+            }
           })}
         </div>
         <div
@@ -46,16 +53,23 @@ function renderDiff(
           aria-labelledby="qaTextHeading"
         >
           ${diff.map((part) => {
-            return html`
+            if (part.added) {
-              <span
+              return html`<ins
-                class=${part.added
+                aria-label="${msg("Added text: Analysis")}"
-                  ? addedText
+                class="${addedText}"
-                  : part.removed
+                >${part.value}</ins
-                    ? removedText
+              >`;
-                    : ""}
+            } else if (part.removed) {
+              return html`<del
+                aria-label="${msg("Missing text: Analysis")}"
+                class="${removedText}"
+                >${part.value}</del
+              >`;
+            } else {
+              return html`<span aria-label="${msg("Identical text")}"
                 >${part.value}</span
-              >
+              >`;
-            `;
+            }
           })}
         </div>
       `;

frontend/src/pages/org/workflow-editor.ts

@@ -57,7 +57,7 @@ import type {
 } from "@/features/crawl-workflows/queue-exclusion-table";
 import { isApiError, type Detail } from "@/utils/api";
 import type { AuthState } from "@/utils/AuthService";
-import { DEFAULT_MAX_SCALE } from "@/utils/crawler";
+import { DEFAULT_MAX_SCALE, DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler";
 import {
   getNextDate,
   getScheduleInterval,
@@ -135,8 +135,6 @@ type FormState = {
   crawlerChannel: string;
 };
 
-const DEPTH_SUPPORTED_SCOPES = ["prefix", "host", "domain", "custom", "any"];
-
 const getDefaultProgressState = (hasConfigId = false): ProgressState => {
   let activeTab: StepName = "crawlSetup";
   if (window.location.hash) {

frontend/src/utils/crawler.ts

@@ -31,6 +31,14 @@ export const inactiveCrawlStates: CrawlState[] = [
 
 export const DEFAULT_MAX_SCALE = 3;
 
+export const DEPTH_SUPPORTED_SCOPES = [
+  "prefix",
+  "host",
+  "domain",
+  "custom",
+  "any",
+];
+
 export function isActive(state: CrawlState | null) {
   return state && activeCrawlStates.includes(state);
 }

frontend/yarn.lock

@@ -1135,10 +1135,10 @@
   resolved "https://registry.yarnpkg.com/@shoelace-style/localize/-/localize-3.1.2.tgz#2c63f16d8aa80842dbe5127845c76ed53f6a5e8e"
   integrity sha512-Hf45HeO+vdQblabpyZOTxJ4ZeZsmIUYXXPmoYrrR4OJ5OKxL+bhMz5mK8JXgl7HsoEowfz7+e248UGi861de9Q==
 
-"@shoelace-style/shoelace@~2.15.0":
+"@shoelace-style/shoelace@~2.15.1":
-  version "2.15.0"
+  version "2.15.1"
-  resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.0.tgz#3410d6bb50811fad001b2c2fbd455cb60d6341a9"
+  resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.1.tgz#2fa6bd8e493801f5b5b4744fab0fa108bbc01934"
-  integrity sha512-Lcg938Y8U2VsHqIYewzlt+H1rbrXC4GRSUkTJgXyF8/0YAOlI+srd5OSfIw+/LYmwLP2Peyh398Kae/6tg4PDA==
+  integrity sha512-3ecUw8gRwOtcZQ8kWWkjk4FTfObYQ/XIl3aRhxprESoOYV1cYhloYPsmQY38UoL3+pwJiZb5+LzX0l3u3Zl0GA==
   dependencies:
     "@ctrl/tinycolor" "^4.0.2"
     "@floating-ui/dom" "^1.5.3"