Ilya Kreymer
GitHub
commit
16a720c685
@ -64,7 +64,8 @@ class BaseCrawlOps:
|
||||
background_job_ops: BackgroundJobOps
|
||||
page_ops: PageOps
|
||||
|
||||
presign_duration: int
|
||||
presign_duration_seconds: int
|
||||
expire_at_duration_seconds: int
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@ -95,6 +96,9 @@ class BaseCrawlOps:
|
||||
min(presign_duration_minutes, PRESIGN_MINUTES_MAX) * 60
|
||||
)
|
||||
|
||||
# renew when <25% of time remaining
|
||||
self.expire_at_duration_seconds = int(self.presign_duration_seconds * 0.75)
|
||||
|
||||
def set_page_ops(self, page_ops):
|
||||
"""set page ops reference"""
|
||||
self.page_ops = page_ops
|
||||
@ -434,7 +438,7 @@ class BaseCrawlOps:
|
||||
print("no files")
|
||||
return []
|
||||
|
||||
delta = timedelta(seconds=self.presign_duration_seconds)
|
||||
delta = timedelta(seconds=self.expire_at_duration_seconds)
|
||||
|
||||
out_files = []
|
||||
|
||||
|
||||
@ -325,8 +325,15 @@ class CrawlOperator(BaseOperator):
|
||||
qa_source_crawl_id = params["qa_source_crawl_id"]
|
||||
name = f"qa-replay-{qa_source_crawl_id}"
|
||||
|
||||
if name in children[CMAP]:
|
||||
return [children[CMAP][name]]
|
||||
configmap = children[CMAP].get(name)
|
||||
if configmap and not self._qa_configmap_update_needed(name, configmap):
|
||||
metadata = configmap["metadata"]
|
||||
configmap["metadata"] = {
|
||||
"name": metadata["name"],
|
||||
"namespace": metadata["namespace"],
|
||||
"labels": metadata["labels"],
|
||||
}
|
||||
return [configmap]
|
||||
|
||||
crawl_replay = await self.crawl_ops.get_internal_crawl_out(qa_source_crawl_id)
|
||||
|
||||
@ -364,6 +371,22 @@ class CrawlOperator(BaseOperator):
|
||||
|
||||
return self.load_from_yaml("crawler.yaml", params)
|
||||
|
||||
def _qa_configmap_update_needed(self, name, configmap):
|
||||
try:
|
||||
now = dt_now()
|
||||
resources = json.loads(configmap["data"]["qa-config.json"])["resources"]
|
||||
for resource in resources:
|
||||
expire_at = datetime.fromisoformat(resource["expireAt"])
|
||||
if expire_at <= now:
|
||||
print(f"Refreshing QA configmap for QA run: {name}")
|
||||
return True
|
||||
|
||||
# pylint: disable=broad-exception-caught
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
return False
|
||||
|
||||
# pylint: disable=too-many-arguments
|
||||
async def _resolve_scale(
|
||||
self,
|
||||
|
||||
@ -124,7 +124,7 @@ spec:
|
||||
- {{ redis_url }}
|
||||
{% if qa_source_crawl_id %}
|
||||
- --qaSource
|
||||
- /tmp/qa-config.json
|
||||
- /tmp/qa/qa-config.json
|
||||
{% elif profile_filename %}
|
||||
- --profile
|
||||
- "@{{ profile_filename }}"
|
||||
@ -137,8 +137,7 @@ spec:
|
||||
|
||||
{% if qa_source_crawl_id %}
|
||||
- name: qa-config
|
||||
mountPath: /tmp/qa-config.json
|
||||
subPath: qa-config.json
|
||||
mountPath: /tmp/qa/
|
||||
readOnly: True
|
||||
{% endif %}
|
||||
|
||||
|
||||
@ -23,7 +23,7 @@ spec:
|
||||
- apiVersion: v1
|
||||
resource: configmaps
|
||||
updateStrategy:
|
||||
method: OnDelete
|
||||
method: InPlace
|
||||
|
||||
hooks:
|
||||
sync:
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
"@lit/task": "^1.0.0",
|
||||
"@novnc/novnc": "^1.4.0-beta",
|
||||
"@rollup/plugin-commonjs": "^18.0.0",
|
||||
"@shoelace-style/shoelace": "~2.15.0",
|
||||
"@shoelace-style/shoelace": "~2.15.1",
|
||||
"@tailwindcss/container-queries": "^0.1.1",
|
||||
"@types/color": "^3.0.2",
|
||||
"@types/diff": "^5.0.9",
|
||||
|
||||
@ -13,6 +13,7 @@ import type { CrawlConfig, Seed, SeedConfig } from "@/pages/org/types";
|
||||
import type { Collection } from "@/types/collection";
|
||||
import { isApiError } from "@/utils/api";
|
||||
import type { AuthState } from "@/utils/AuthService";
|
||||
import { DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler";
|
||||
import { humanizeSchedule } from "@/utils/cron";
|
||||
import LiteElement, { html } from "@/utils/LiteElement";
|
||||
|
||||
@ -141,6 +142,9 @@ export class ConfigDetails extends LiteElement {
|
||||
`,
|
||||
() => this.renderSetting(msg("Exclusions"), msg("None")),
|
||||
)}
|
||||
<btrix-section-heading style="--margin: var(--sl-spacing-medium)">
|
||||
<h4>${msg("Per-Crawl Limits")}</h4>
|
||||
</btrix-section-heading>
|
||||
${this.renderSetting(
|
||||
msg("Max Pages"),
|
||||
when(
|
||||
@ -157,6 +161,21 @@ export class ConfigDetails extends LiteElement {
|
||||
: undefined,
|
||||
),
|
||||
)}
|
||||
${this.renderSetting(
|
||||
msg("Crawl Time Limit"),
|
||||
renderTimeLimit(crawlConfig?.crawlTimeout, Infinity),
|
||||
)}
|
||||
${this.renderSetting(
|
||||
msg("Crawl Size Limit"),
|
||||
renderSize(crawlConfig?.maxCrawlSize),
|
||||
)}
|
||||
${this.renderSetting(
|
||||
msg("Crawler Instances"),
|
||||
crawlConfig?.scale ? `${crawlConfig.scale}×` : "",
|
||||
)}
|
||||
<btrix-section-heading style="--margin: var(--sl-spacing-medium)">
|
||||
<h4>${msg("Per-Page Limits")}</h4>
|
||||
</btrix-section-heading>
|
||||
${this.renderSetting(
|
||||
msg("Page Load Timeout"),
|
||||
renderTimeLimit(
|
||||
@ -169,7 +188,7 @@ export class ConfigDetails extends LiteElement {
|
||||
renderTimeLimit(crawlConfig?.config.postLoadDelay, 0),
|
||||
)}
|
||||
${this.renderSetting(
|
||||
msg("Page Behavior Timeout"),
|
||||
msg("Behavior Timeout"),
|
||||
renderTimeLimit(
|
||||
crawlConfig?.config.behaviorTimeout,
|
||||
this.orgDefaults?.behaviorTimeoutSeconds ?? Infinity,
|
||||
@ -188,18 +207,6 @@ export class ConfigDetails extends LiteElement {
|
||||
msg("Delay Before Next Page"),
|
||||
renderTimeLimit(crawlConfig?.config.pageExtraDelay, 0),
|
||||
)}
|
||||
${this.renderSetting(
|
||||
msg("Crawl Time Limit"),
|
||||
renderTimeLimit(crawlConfig?.crawlTimeout, Infinity),
|
||||
)}
|
||||
${this.renderSetting(
|
||||
msg("Crawl Size Limit"),
|
||||
renderSize(crawlConfig?.maxCrawlSize),
|
||||
)}
|
||||
${this.renderSetting(
|
||||
msg("Crawler Instances"),
|
||||
crawlConfig?.scale ? `${crawlConfig.scale}×` : "",
|
||||
)}
|
||||
</btrix-desc-list>
|
||||
</section>
|
||||
<section id="browser-settings" class="mb-8">
|
||||
@ -221,7 +228,11 @@ export class ConfigDetails extends LiteElement {
|
||||
>
|
||||
${crawlConfig?.profileName}
|
||||
</a>`,
|
||||
() => crawlConfig?.profileName || msg("Default Profile"),
|
||||
() =>
|
||||
crawlConfig?.profileName ||
|
||||
html`<span class="text-neutral-400"
|
||||
>${msg("Default Profile")}</span
|
||||
>`,
|
||||
),
|
||||
)}
|
||||
${this.renderSetting(
|
||||
@ -237,7 +248,7 @@ export class ConfigDetails extends LiteElement {
|
||||
msg("User Agent"),
|
||||
crawlConfig?.config.userAgent
|
||||
? crawlConfig.config.userAgent
|
||||
: msg("Default User Agent"),
|
||||
: html`<span class="text-neutral-400">${msg("Default")}</span>`,
|
||||
)}
|
||||
${crawlConfig?.config.lang
|
||||
? this.renderSetting(
|
||||
@ -396,15 +407,15 @@ export class ConfigDetails extends LiteElement {
|
||||
true,
|
||||
)}
|
||||
${when(
|
||||
["host", "domain", "custom", "any"].includes(
|
||||
DEPTH_SUPPORTED_SCOPES.includes(
|
||||
primarySeedConfig!.scopeType || seedsConfig.scopeType!,
|
||||
),
|
||||
() =>
|
||||
this.renderSetting(
|
||||
msg("Max Depth"),
|
||||
primarySeedConfig?.depth
|
||||
primarySeedConfig && primarySeedConfig.depth !== null
|
||||
? msg(str`${primarySeedConfig.depth} hop(s)`)
|
||||
: msg("None"),
|
||||
: msg("Unlimited (default)"),
|
||||
),
|
||||
)}
|
||||
${this.renderSetting(
|
||||
|
||||
@ -45,6 +45,14 @@ export class ArchivedItemListItem extends TailwindElement {
|
||||
var(--btrix-border-radius-bottom, 0);
|
||||
height: 2.5rem;
|
||||
}
|
||||
|
||||
sl-progress-ring {
|
||||
/* Setting size to var(--font-size-base) breaks in chrome,
|
||||
have cell contents inherit size from cell instead */
|
||||
--size: 1em;
|
||||
--track-width: 1px;
|
||||
--indicator-width: 2px;
|
||||
}
|
||||
`;
|
||||
|
||||
@property({ type: Object, attribute: false })
|
||||
@ -141,7 +149,7 @@ export class ArchivedItemListItem extends TailwindElement {
|
||||
</btrix-table-cell>
|
||||
`
|
||||
: nothing}
|
||||
<btrix-table-cell class="pr-0">
|
||||
<btrix-table-cell class="pr-0 text-base">
|
||||
${this.showStatus
|
||||
? html`
|
||||
<btrix-crawl-status
|
||||
@ -158,7 +166,7 @@ export class ArchivedItemListItem extends TailwindElement {
|
||||
hoist
|
||||
>
|
||||
<sl-icon
|
||||
class="text-base"
|
||||
class="text-inherit"
|
||||
style="color: ${crawlStatus.cssColor}"
|
||||
name=${typeIcon}
|
||||
label=${typeLabel}
|
||||
@ -179,12 +187,12 @@ export class ArchivedItemListItem extends TailwindElement {
|
||||
? html`
|
||||
<sl-progress-ring
|
||||
value="${activeProgress}"
|
||||
style="color: ${qaStatus.cssColor}; --size: var(--font-size-base); --track-width: 1px; --indicator-width: 2px;"
|
||||
style="color: ${qaStatus.cssColor};"
|
||||
></sl-progress-ring>
|
||||
`
|
||||
: html`
|
||||
<sl-icon
|
||||
class="text-base"
|
||||
class="text-inherit"
|
||||
style="color: ${qaStatus.cssColor}"
|
||||
name=${isUpload ? "slash" : "microscope"}
|
||||
library=${isUpload ? "default" : "app"}
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
/>
|
||||
<title>Browsertrix</title>
|
||||
<base href="/" />
|
||||
<script defer src="/replay/ui.js"></script>
|
||||
|
||||
<script
|
||||
src="https://browser.sentry-cdn.com/5.5.0/bundle.min.js"
|
||||
crossorigin="anonymous"
|
||||
@ -38,5 +38,14 @@
|
||||
});
|
||||
}
|
||||
</script>
|
||||
<script>
|
||||
customElements.whenDefined("browsertrix-app").then(() => {
|
||||
// Load replay UI after browsertrix app is defined to prevent issues
|
||||
// with re-defining shoelace components
|
||||
const script = document.createElement("script");
|
||||
script.src = "/replay/ui.js";
|
||||
document.body.appendChild(script);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@ -440,7 +440,6 @@ export class ArchivedItemDetailQA extends TailwindElement {
|
||||
<div slot="footer" class="flex justify-between">
|
||||
<sl-button
|
||||
size="small"
|
||||
.autofocus=${true}
|
||||
@click=${() => void this.deleteQADialog?.hide()}
|
||||
>
|
||||
${msg("Cancel")}
|
||||
|
||||
@ -508,6 +508,7 @@ export class ArchivedItemQA extends TailwindElement {
|
||||
|
||||
<div class="grid--tabGroup flex min-w-0 flex-col">
|
||||
<nav
|
||||
aria-label="${msg("Page heuristics")}"
|
||||
class="-mx-3 my-0 flex gap-2 overflow-x-auto px-3 py-2 lg:mx-0 lg:px-0"
|
||||
>
|
||||
<btrix-navigation-button
|
||||
|
||||
@ -20,7 +20,7 @@ function renderDiff(
|
||||
diffImport.then(({ diffWords }) => {
|
||||
const diff = diffWords(crawlText, qaText);
|
||||
|
||||
const addedText = tw`bg-red-100 text-red-700`;
|
||||
const addedText = tw`bg-red-100 text-red-700 no-underline`;
|
||||
const removedText = tw`bg-red-100 text-red-100`;
|
||||
|
||||
return html`
|
||||
@ -29,16 +29,23 @@ function renderDiff(
|
||||
aria-labelledby="crawlTextHeading"
|
||||
>
|
||||
${diff.map((part) => {
|
||||
return html`
|
||||
<span
|
||||
class=${part.added
|
||||
? removedText
|
||||
: part.removed
|
||||
? addedText
|
||||
: ""}
|
||||
if (part.added) {
|
||||
return html`<del
|
||||
aria-label="${msg("Missing text: Crawl")}"
|
||||
class="${removedText}"
|
||||
>${part.value}</del
|
||||
>`;
|
||||
} else if (part.removed) {
|
||||
return html`<ins
|
||||
aria-label="${msg("Added text: Crawl")}"
|
||||
class="${addedText}"
|
||||
>${part.value}</ins
|
||||
>`;
|
||||
} else {
|
||||
return html`<span aria-label="${msg("Identical text")}"
|
||||
>${part.value}</span
|
||||
>
|
||||
`;
|
||||
>`;
|
||||
}
|
||||
})}
|
||||
</div>
|
||||
<div
|
||||
@ -46,16 +53,23 @@ function renderDiff(
|
||||
aria-labelledby="qaTextHeading"
|
||||
>
|
||||
${diff.map((part) => {
|
||||
return html`
|
||||
<span
|
||||
class=${part.added
|
||||
? addedText
|
||||
: part.removed
|
||||
? removedText
|
||||
: ""}
|
||||
if (part.added) {
|
||||
return html`<ins
|
||||
aria-label="${msg("Added text: Analysis")}"
|
||||
class="${addedText}"
|
||||
>${part.value}</ins
|
||||
>`;
|
||||
} else if (part.removed) {
|
||||
return html`<del
|
||||
aria-label="${msg("Missing text: Analysis")}"
|
||||
class="${removedText}"
|
||||
>${part.value}</del
|
||||
>`;
|
||||
} else {
|
||||
return html`<span aria-label="${msg("Identical text")}"
|
||||
>${part.value}</span
|
||||
>
|
||||
`;
|
||||
>`;
|
||||
}
|
||||
})}
|
||||
</div>
|
||||
`;
|
||||
|
||||
@ -57,7 +57,7 @@ import type {
|
||||
} from "@/features/crawl-workflows/queue-exclusion-table";
|
||||
import { isApiError, type Detail } from "@/utils/api";
|
||||
import type { AuthState } from "@/utils/AuthService";
|
||||
import { DEFAULT_MAX_SCALE } from "@/utils/crawler";
|
||||
import { DEFAULT_MAX_SCALE, DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler";
|
||||
import {
|
||||
getNextDate,
|
||||
getScheduleInterval,
|
||||
@ -135,8 +135,6 @@ type FormState = {
|
||||
crawlerChannel: string;
|
||||
};
|
||||
|
||||
const DEPTH_SUPPORTED_SCOPES = ["prefix", "host", "domain", "custom", "any"];
|
||||
|
||||
const getDefaultProgressState = (hasConfigId = false): ProgressState => {
|
||||
let activeTab: StepName = "crawlSetup";
|
||||
if (window.location.hash) {
|
||||
|
||||
@ -31,6 +31,14 @@ export const inactiveCrawlStates: CrawlState[] = [
|
||||
|
||||
export const DEFAULT_MAX_SCALE = 3;
|
||||
|
||||
export const DEPTH_SUPPORTED_SCOPES = [
|
||||
"prefix",
|
||||
"host",
|
||||
"domain",
|
||||
"custom",
|
||||
"any",
|
||||
];
|
||||
|
||||
export function isActive(state: CrawlState | null) {
|
||||
return state && activeCrawlStates.includes(state);
|
||||
}
|
||||
|
||||
@ -1135,10 +1135,10 @@
|
||||
resolved "https://registry.yarnpkg.com/@shoelace-style/localize/-/localize-3.1.2.tgz#2c63f16d8aa80842dbe5127845c76ed53f6a5e8e"
|
||||
integrity sha512-Hf45HeO+vdQblabpyZOTxJ4ZeZsmIUYXXPmoYrrR4OJ5OKxL+bhMz5mK8JXgl7HsoEowfz7+e248UGi861de9Q==
|
||||
|
||||
"@shoelace-style/shoelace@~2.15.0":
|
||||
version "2.15.0"
|
||||
resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.0.tgz#3410d6bb50811fad001b2c2fbd455cb60d6341a9"
|
||||
integrity sha512-Lcg938Y8U2VsHqIYewzlt+H1rbrXC4GRSUkTJgXyF8/0YAOlI+srd5OSfIw+/LYmwLP2Peyh398Kae/6tg4PDA==
|
||||
"@shoelace-style/shoelace@~2.15.1":
|
||||
version "2.15.1"
|
||||
resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.1.tgz#2fa6bd8e493801f5b5b4744fab0fa108bbc01934"
|
||||
integrity sha512-3ecUw8gRwOtcZQ8kWWkjk4FTfObYQ/XIl3aRhxprESoOYV1cYhloYPsmQY38UoL3+pwJiZb5+LzX0l3u3Zl0GA==
|
||||
dependencies:
|
||||
"@ctrl/tinycolor" "^4.0.2"
|
||||
"@floating-ui/dom" "^1.5.3"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user