commit
						16a720c685
					
				| @ -64,7 +64,8 @@ class BaseCrawlOps: | ||||
|     background_job_ops: BackgroundJobOps | ||||
|     page_ops: PageOps | ||||
| 
 | ||||
|     presign_duration: int | ||||
|     presign_duration_seconds: int | ||||
|     expire_at_duration_seconds: int | ||||
| 
 | ||||
|     def __init__( | ||||
|         self, | ||||
| @ -95,6 +96,9 @@ class BaseCrawlOps: | ||||
|             min(presign_duration_minutes, PRESIGN_MINUTES_MAX) * 60 | ||||
|         ) | ||||
| 
 | ||||
|         # renew when <25% of time remaining | ||||
|         self.expire_at_duration_seconds = int(self.presign_duration_seconds * 0.75) | ||||
| 
 | ||||
|     def set_page_ops(self, page_ops): | ||||
|         """set page ops reference""" | ||||
|         self.page_ops = page_ops | ||||
| @ -434,7 +438,7 @@ class BaseCrawlOps: | ||||
|             print("no files") | ||||
|             return [] | ||||
| 
 | ||||
|         delta = timedelta(seconds=self.presign_duration_seconds) | ||||
|         delta = timedelta(seconds=self.expire_at_duration_seconds) | ||||
| 
 | ||||
|         out_files = [] | ||||
| 
 | ||||
|  | ||||
| @ -325,8 +325,15 @@ class CrawlOperator(BaseOperator): | ||||
|         qa_source_crawl_id = params["qa_source_crawl_id"] | ||||
|         name = f"qa-replay-{qa_source_crawl_id}" | ||||
| 
 | ||||
|         if name in children[CMAP]: | ||||
|             return [children[CMAP][name]] | ||||
|         configmap = children[CMAP].get(name) | ||||
|         if configmap and not self._qa_configmap_update_needed(name, configmap): | ||||
|             metadata = configmap["metadata"] | ||||
|             configmap["metadata"] = { | ||||
|                 "name": metadata["name"], | ||||
|                 "namespace": metadata["namespace"], | ||||
|                 "labels": metadata["labels"], | ||||
|             } | ||||
|             return [configmap] | ||||
| 
 | ||||
|         crawl_replay = await self.crawl_ops.get_internal_crawl_out(qa_source_crawl_id) | ||||
| 
 | ||||
| @ -364,6 +371,22 @@ class CrawlOperator(BaseOperator): | ||||
| 
 | ||||
|         return self.load_from_yaml("crawler.yaml", params) | ||||
| 
 | ||||
|     def _qa_configmap_update_needed(self, name, configmap): | ||||
|         try: | ||||
|             now = dt_now() | ||||
|             resources = json.loads(configmap["data"]["qa-config.json"])["resources"] | ||||
|             for resource in resources: | ||||
|                 expire_at = datetime.fromisoformat(resource["expireAt"]) | ||||
|                 if expire_at <= now: | ||||
|                     print(f"Refreshing QA configmap for QA run: {name}") | ||||
|                     return True | ||||
| 
 | ||||
|         # pylint: disable=broad-exception-caught | ||||
|         except Exception as e: | ||||
|             print(e) | ||||
| 
 | ||||
|         return False | ||||
| 
 | ||||
|     # pylint: disable=too-many-arguments | ||||
|     async def _resolve_scale( | ||||
|         self, | ||||
|  | ||||
| @ -124,7 +124,7 @@ spec: | ||||
|         - {{ redis_url }} | ||||
|       {% if qa_source_crawl_id %} | ||||
|         - --qaSource | ||||
|         - /tmp/qa-config.json | ||||
|         - /tmp/qa/qa-config.json | ||||
|       {% elif profile_filename %} | ||||
|         - --profile | ||||
|         - "@{{ profile_filename }}" | ||||
| @ -137,8 +137,7 @@ spec: | ||||
| 
 | ||||
|       {% if qa_source_crawl_id %} | ||||
|         - name: qa-config | ||||
|           mountPath: /tmp/qa-config.json | ||||
|           subPath: qa-config.json | ||||
|           mountPath: /tmp/qa/ | ||||
|           readOnly: True | ||||
|       {% endif %} | ||||
| 
 | ||||
|  | ||||
| @ -23,7 +23,7 @@ spec: | ||||
|   - apiVersion: v1 | ||||
|     resource: configmaps | ||||
|     updateStrategy: | ||||
|       method: OnDelete | ||||
|       method: InPlace | ||||
| 
 | ||||
|   hooks: | ||||
|     sync: | ||||
|  | ||||
| @ -10,7 +10,7 @@ | ||||
|     "@lit/task": "^1.0.0", | ||||
|     "@novnc/novnc": "^1.4.0-beta", | ||||
|     "@rollup/plugin-commonjs": "^18.0.0", | ||||
|     "@shoelace-style/shoelace": "~2.15.0", | ||||
|     "@shoelace-style/shoelace": "~2.15.1", | ||||
|     "@tailwindcss/container-queries": "^0.1.1", | ||||
|     "@types/color": "^3.0.2", | ||||
|     "@types/diff": "^5.0.9", | ||||
|  | ||||
| @ -13,6 +13,7 @@ import type { CrawlConfig, Seed, SeedConfig } from "@/pages/org/types"; | ||||
| import type { Collection } from "@/types/collection"; | ||||
| import { isApiError } from "@/utils/api"; | ||||
| import type { AuthState } from "@/utils/AuthService"; | ||||
| import { DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler"; | ||||
| import { humanizeSchedule } from "@/utils/cron"; | ||||
| import LiteElement, { html } from "@/utils/LiteElement"; | ||||
| 
 | ||||
| @ -141,6 +142,9 @@ export class ConfigDetails extends LiteElement { | ||||
|             `,
 | ||||
|             () => this.renderSetting(msg("Exclusions"), msg("None")), | ||||
|           )} | ||||
|           <btrix-section-heading style="--margin: var(--sl-spacing-medium)"> | ||||
|             <h4>${msg("Per-Crawl Limits")}</h4> | ||||
|           </btrix-section-heading> | ||||
|           ${this.renderSetting( | ||||
|             msg("Max Pages"), | ||||
|             when( | ||||
| @ -157,6 +161,21 @@ export class ConfigDetails extends LiteElement { | ||||
|                   : undefined, | ||||
|             ), | ||||
|           )} | ||||
|           ${this.renderSetting( | ||||
|             msg("Crawl Time Limit"), | ||||
|             renderTimeLimit(crawlConfig?.crawlTimeout, Infinity), | ||||
|           )} | ||||
|           ${this.renderSetting( | ||||
|             msg("Crawl Size Limit"), | ||||
|             renderSize(crawlConfig?.maxCrawlSize), | ||||
|           )} | ||||
|           ${this.renderSetting( | ||||
|             msg("Crawler Instances"), | ||||
|             crawlConfig?.scale ? `${crawlConfig.scale}×` : "", | ||||
|           )} | ||||
|           <btrix-section-heading style="--margin: var(--sl-spacing-medium)"> | ||||
|             <h4>${msg("Per-Page Limits")}</h4> | ||||
|           </btrix-section-heading> | ||||
|           ${this.renderSetting( | ||||
|             msg("Page Load Timeout"), | ||||
|             renderTimeLimit( | ||||
| @ -169,7 +188,7 @@ export class ConfigDetails extends LiteElement { | ||||
|             renderTimeLimit(crawlConfig?.config.postLoadDelay, 0), | ||||
|           )} | ||||
|           ${this.renderSetting( | ||||
|             msg("Page Behavior Timeout"), | ||||
|             msg("Behavior Timeout"), | ||||
|             renderTimeLimit( | ||||
|               crawlConfig?.config.behaviorTimeout, | ||||
|               this.orgDefaults?.behaviorTimeoutSeconds ?? Infinity, | ||||
| @ -188,18 +207,6 @@ export class ConfigDetails extends LiteElement { | ||||
|             msg("Delay Before Next Page"), | ||||
|             renderTimeLimit(crawlConfig?.config.pageExtraDelay, 0), | ||||
|           )} | ||||
|           ${this.renderSetting( | ||||
|             msg("Crawl Time Limit"), | ||||
|             renderTimeLimit(crawlConfig?.crawlTimeout, Infinity), | ||||
|           )} | ||||
|           ${this.renderSetting( | ||||
|             msg("Crawl Size Limit"), | ||||
|             renderSize(crawlConfig?.maxCrawlSize), | ||||
|           )} | ||||
|           ${this.renderSetting( | ||||
|             msg("Crawler Instances"), | ||||
|             crawlConfig?.scale ? `${crawlConfig.scale}×` : "", | ||||
|           )} | ||||
|         </btrix-desc-list> | ||||
|       </section> | ||||
|       <section id="browser-settings" class="mb-8"> | ||||
| @ -221,7 +228,11 @@ export class ConfigDetails extends LiteElement { | ||||
|                 > | ||||
|                   ${crawlConfig?.profileName} | ||||
|                 </a>`,
 | ||||
|               () => crawlConfig?.profileName || msg("Default Profile"), | ||||
|               () => | ||||
|                 crawlConfig?.profileName || | ||||
|                 html`<span class="text-neutral-400"
 | ||||
|                   >${msg("Default Profile")}</span | ||||
|                 >`,
 | ||||
|             ), | ||||
|           )} | ||||
|           ${this.renderSetting( | ||||
| @ -237,7 +248,7 @@ export class ConfigDetails extends LiteElement { | ||||
|             msg("User Agent"), | ||||
|             crawlConfig?.config.userAgent | ||||
|               ? crawlConfig.config.userAgent | ||||
|               : msg("Default User Agent"), | ||||
|               : html`<span class="text-neutral-400">${msg("Default")}</span>`, | ||||
|           )} | ||||
|           ${crawlConfig?.config.lang | ||||
|             ? this.renderSetting( | ||||
| @ -396,15 +407,15 @@ export class ConfigDetails extends LiteElement { | ||||
|         true, | ||||
|       )} | ||||
|       ${when( | ||||
|         ["host", "domain", "custom", "any"].includes( | ||||
|         DEPTH_SUPPORTED_SCOPES.includes( | ||||
|           primarySeedConfig!.scopeType || seedsConfig.scopeType!, | ||||
|         ), | ||||
|         () => | ||||
|           this.renderSetting( | ||||
|             msg("Max Depth"), | ||||
|             primarySeedConfig?.depth | ||||
|             primarySeedConfig && primarySeedConfig.depth !== null | ||||
|               ? msg(str`${primarySeedConfig.depth} hop(s)`) | ||||
|               : msg("None"), | ||||
|               : msg("Unlimited (default)"), | ||||
|           ), | ||||
|       )} | ||||
|       ${this.renderSetting( | ||||
|  | ||||
| @ -45,6 +45,14 @@ export class ArchivedItemListItem extends TailwindElement { | ||||
|         var(--btrix-border-radius-bottom, 0); | ||||
|       height: 2.5rem; | ||||
|     } | ||||
| 
 | ||||
|     sl-progress-ring { | ||||
|       /* Setting size to var(--font-size-base) breaks in chrome, | ||||
|       have cell contents inherit size from cell instead */ | ||||
|       --size: 1em; | ||||
|       --track-width: 1px; | ||||
|       --indicator-width: 2px; | ||||
|     } | ||||
|   `;
 | ||||
| 
 | ||||
|   @property({ type: Object, attribute: false }) | ||||
| @ -141,7 +149,7 @@ export class ArchivedItemListItem extends TailwindElement { | ||||
|               </btrix-table-cell> | ||||
|             ` | ||||
|           : nothing} | ||||
|         <btrix-table-cell class="pr-0"> | ||||
|         <btrix-table-cell class="pr-0 text-base"> | ||||
|           ${this.showStatus | ||||
|             ? html` | ||||
|                 <btrix-crawl-status | ||||
| @ -158,7 +166,7 @@ export class ArchivedItemListItem extends TailwindElement { | ||||
|                   hoist | ||||
|                 > | ||||
|                   <sl-icon | ||||
|                     class="text-base" | ||||
|                     class="text-inherit" | ||||
|                     style="color: ${crawlStatus.cssColor}" | ||||
|                     name=${typeIcon} | ||||
|                     label=${typeLabel} | ||||
| @ -179,12 +187,12 @@ export class ArchivedItemListItem extends TailwindElement { | ||||
|               ? html` | ||||
|                   <sl-progress-ring | ||||
|                     value="${activeProgress}" | ||||
|                     style="color: ${qaStatus.cssColor}; --size: var(--font-size-base); --track-width: 1px; --indicator-width: 2px;" | ||||
|                     style="color: ${qaStatus.cssColor};" | ||||
|                   ></sl-progress-ring> | ||||
|                 ` | ||||
|               : html` | ||||
|                   <sl-icon | ||||
|                     class="text-base" | ||||
|                     class="text-inherit" | ||||
|                     style="color: ${qaStatus.cssColor}" | ||||
|                     name=${isUpload ? "slash" : "microscope"} | ||||
|                     library=${isUpload ? "default" : "app"} | ||||
|  | ||||
| @ -8,7 +8,7 @@ | ||||
|     /> | ||||
|     <title>Browsertrix</title> | ||||
|     <base href="/" /> | ||||
|     <script defer src="/replay/ui.js"></script> | ||||
| 
 | ||||
|     <script | ||||
|       src="https://browser.sentry-cdn.com/5.5.0/bundle.min.js" | ||||
|       crossorigin="anonymous" | ||||
| @ -38,5 +38,14 @@ | ||||
|         }); | ||||
|       } | ||||
|     </script> | ||||
|     <script> | ||||
|       customElements.whenDefined("browsertrix-app").then(() => { | ||||
|         // Load replay UI after browsertrix app is defined to prevent issues | ||||
|         // with re-defining shoelace components | ||||
|         const script = document.createElement("script"); | ||||
|         script.src = "/replay/ui.js"; | ||||
|         document.body.appendChild(script); | ||||
|       }); | ||||
|     </script> | ||||
|   </body> | ||||
| </html> | ||||
|  | ||||
| @ -440,7 +440,6 @@ export class ArchivedItemDetailQA extends TailwindElement { | ||||
|           <div slot="footer" class="flex justify-between"> | ||||
|             <sl-button | ||||
|               size="small" | ||||
|               .autofocus=${true} | ||||
|               @click=${() => void this.deleteQADialog?.hide()} | ||||
|             > | ||||
|               ${msg("Cancel")} | ||||
|  | ||||
| @ -508,6 +508,7 @@ export class ArchivedItemQA extends TailwindElement { | ||||
| 
 | ||||
|         <div class="grid--tabGroup flex min-w-0 flex-col"> | ||||
|           <nav | ||||
|             aria-label="${msg("Page heuristics")}" | ||||
|             class="-mx-3 my-0 flex gap-2 overflow-x-auto px-3 py-2 lg:mx-0 lg:px-0" | ||||
|           > | ||||
|             <btrix-navigation-button | ||||
|  | ||||
| @ -20,7 +20,7 @@ function renderDiff( | ||||
|     diffImport.then(({ diffWords }) => { | ||||
|       const diff = diffWords(crawlText, qaText); | ||||
| 
 | ||||
|       const addedText = tw`bg-red-100 text-red-700`; | ||||
|       const addedText = tw`bg-red-100 text-red-700 no-underline`; | ||||
|       const removedText = tw`bg-red-100 text-red-100`; | ||||
| 
 | ||||
|       return html` | ||||
| @ -29,16 +29,23 @@ function renderDiff( | ||||
|           aria-labelledby="crawlTextHeading" | ||||
|         > | ||||
|           ${diff.map((part) => { | ||||
|             return html` | ||||
|               <span | ||||
|                 class=${part.added | ||||
|                   ? removedText | ||||
|                   : part.removed | ||||
|                     ? addedText | ||||
|                     : ""} | ||||
|             if (part.added) { | ||||
|               return html`<del
 | ||||
|                 aria-label="${msg("Missing text: Crawl")}" | ||||
|                 class="${removedText}" | ||||
|                 >${part.value}</del | ||||
|               >`;
 | ||||
|             } else if (part.removed) { | ||||
|               return html`<ins
 | ||||
|                 aria-label="${msg("Added text: Crawl")}" | ||||
|                 class="${addedText}" | ||||
|                 >${part.value}</ins | ||||
|               >`;
 | ||||
|             } else { | ||||
|               return html`<span aria-label="${msg("Identical text")}"
 | ||||
|                 >${part.value}</span | ||||
|               > | ||||
|             `;
 | ||||
|               >`;
 | ||||
|             } | ||||
|           })} | ||||
|         </div> | ||||
|         <div | ||||
| @ -46,16 +53,23 @@ function renderDiff( | ||||
|           aria-labelledby="qaTextHeading" | ||||
|         > | ||||
|           ${diff.map((part) => { | ||||
|             return html` | ||||
|               <span | ||||
|                 class=${part.added | ||||
|                   ? addedText | ||||
|                   : part.removed | ||||
|                     ? removedText | ||||
|                     : ""} | ||||
|             if (part.added) { | ||||
|               return html`<ins
 | ||||
|                 aria-label="${msg("Added text: Analysis")}" | ||||
|                 class="${addedText}" | ||||
|                 >${part.value}</ins | ||||
|               >`;
 | ||||
|             } else if (part.removed) { | ||||
|               return html`<del
 | ||||
|                 aria-label="${msg("Missing text: Analysis")}" | ||||
|                 class="${removedText}" | ||||
|                 >${part.value}</del | ||||
|               >`;
 | ||||
|             } else { | ||||
|               return html`<span aria-label="${msg("Identical text")}"
 | ||||
|                 >${part.value}</span | ||||
|               > | ||||
|             `;
 | ||||
|               >`;
 | ||||
|             } | ||||
|           })} | ||||
|         </div> | ||||
|       `;
 | ||||
|  | ||||
| @ -57,7 +57,7 @@ import type { | ||||
| } from "@/features/crawl-workflows/queue-exclusion-table"; | ||||
| import { isApiError, type Detail } from "@/utils/api"; | ||||
| import type { AuthState } from "@/utils/AuthService"; | ||||
| import { DEFAULT_MAX_SCALE } from "@/utils/crawler"; | ||||
| import { DEFAULT_MAX_SCALE, DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler"; | ||||
| import { | ||||
|   getNextDate, | ||||
|   getScheduleInterval, | ||||
| @ -135,8 +135,6 @@ type FormState = { | ||||
|   crawlerChannel: string; | ||||
| }; | ||||
| 
 | ||||
| const DEPTH_SUPPORTED_SCOPES = ["prefix", "host", "domain", "custom", "any"]; | ||||
| 
 | ||||
| const getDefaultProgressState = (hasConfigId = false): ProgressState => { | ||||
|   let activeTab: StepName = "crawlSetup"; | ||||
|   if (window.location.hash) { | ||||
|  | ||||
| @ -31,6 +31,14 @@ export const inactiveCrawlStates: CrawlState[] = [ | ||||
| 
 | ||||
| export const DEFAULT_MAX_SCALE = 3; | ||||
| 
 | ||||
| export const DEPTH_SUPPORTED_SCOPES = [ | ||||
|   "prefix", | ||||
|   "host", | ||||
|   "domain", | ||||
|   "custom", | ||||
|   "any", | ||||
| ]; | ||||
| 
 | ||||
| export function isActive(state: CrawlState | null) { | ||||
|   return state && activeCrawlStates.includes(state); | ||||
| } | ||||
|  | ||||
| @ -1135,10 +1135,10 @@ | ||||
|   resolved "https://registry.yarnpkg.com/@shoelace-style/localize/-/localize-3.1.2.tgz#2c63f16d8aa80842dbe5127845c76ed53f6a5e8e" | ||||
|   integrity sha512-Hf45HeO+vdQblabpyZOTxJ4ZeZsmIUYXXPmoYrrR4OJ5OKxL+bhMz5mK8JXgl7HsoEowfz7+e248UGi861de9Q== | ||||
| 
 | ||||
| "@shoelace-style/shoelace@~2.15.0": | ||||
|   version "2.15.0" | ||||
|   resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.0.tgz#3410d6bb50811fad001b2c2fbd455cb60d6341a9" | ||||
|   integrity sha512-Lcg938Y8U2VsHqIYewzlt+H1rbrXC4GRSUkTJgXyF8/0YAOlI+srd5OSfIw+/LYmwLP2Peyh398Kae/6tg4PDA== | ||||
| "@shoelace-style/shoelace@~2.15.1": | ||||
|   version "2.15.1" | ||||
|   resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.1.tgz#2fa6bd8e493801f5b5b4744fab0fa108bbc01934" | ||||
|   integrity sha512-3ecUw8gRwOtcZQ8kWWkjk4FTfObYQ/XIl3aRhxprESoOYV1cYhloYPsmQY38UoL3+pwJiZb5+LzX0l3u3Zl0GA== | ||||
|   dependencies: | ||||
|     "@ctrl/tinycolor" "^4.0.2" | ||||
|     "@floating-ui/dom" "^1.5.3" | ||||
|  | ||||
		Loading…
	
		Reference in New Issue
	
	Block a user