import { localized, msg, str } from "@lit/localize"; import type { SlTextarea } from "@shoelace-style/shoelace"; import { serialize } from "@shoelace-style/shoelace/dist/utilities/form.js"; import { merge } from "immutable"; import { html, nothing, type PropertyValues } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; import { cache } from "lit/directives/cache.js"; import { choose } from "lit/directives/choose.js"; import { guard } from "lit/directives/guard.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { until } from "lit/directives/until.js"; import { when } from "lit/directives/when.js"; import queryString from "query-string"; import { styles } from "./styles"; import type * as QATypes from "./types"; import { renderReplay } from "./ui/replay"; import { renderResources } from "./ui/resources"; import { renderScreenshots } from "./ui/screenshots"; import { renderSeverityBadge } from "./ui/severityBadge"; import { renderText } from "./ui/text"; import { TailwindElement } from "@/classes/TailwindElement"; import type { Dialog } from "@/components/ui/dialog"; import { APIController } from "@/controllers/api"; import { NavigateController } from "@/controllers/navigate"; import { NotifyController } from "@/controllers/notify"; import { type QaFilterChangeDetail, type QaPaginationChangeDetail, type QaSortChangeDetail, type SortableFieldNames, type SortDirection, } from "@/features/qa/page-list/page-list"; import { type UpdatePageApprovalDetail } from "@/features/qa/page-qa-approval"; import type { SelectDetail } from "@/features/qa/qa-run-dropdown"; import type { APIPaginatedList, APIPaginationQuery, APISortQuery, } from "@/types/api"; import type { ArchivedItem, ArchivedItemPageComment } from "@/types/crawler"; import type { ArchivedItemQAPage, QARun } from "@/types/qa"; import { type AuthState } from "@/utils/AuthService"; import { finishedCrawlStates, isActive, renderName } from "@/utils/crawler"; import { formatISODateString, getLocale } from "@/utils/localization"; const DEFAULT_PAGE_SIZE = 100; type PageResource = { status?: number; mime?: string; type?: string; }; // From https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types const IMG_EXTS = [ "apng", "avif", "gif", "jpg", "jpeg", "jfif", "pjpeg", "pjp", "png", "svg", "webp", "tif", "tiff", "bmp", "ico", "cur", ]; const tabToPrefix: Record = { screenshots: "view", text: "text", resources: "pageinfo", replay: "", }; @localized() @customElement("btrix-archived-item-qa") export class ArchivedItemQA extends TailwindElement { static styles = styles; @property({ type: Object }) authState?: AuthState; @property({ type: String }) orgId?: string; @property({ type: String }) itemId?: string; @property({ type: String }) itemPageId?: string; @property({ type: String }) qaRunId?: string; @property({ type: String }) tab: QATypes.QATab = "screenshots"; @state() private item?: ArchivedItem; @state() finishedQARuns: | (QARun & { state: (typeof finishedCrawlStates)[number] })[] | undefined = []; @state() private pages?: APIPaginatedList; @property({ type: Object }) page?: ArchivedItemQAPage; @state() private crawlData: QATypes.ReplayData = null; @state() private qaData: QATypes.ReplayData = null; // indicate whether the crawl / qa endpoints have been registered in SW // if not, requires loading via // endpoints may be registered but crawlData / qaData may still be missing @state() private crawlDataRegistered = false; @state() private qaDataRegistered = false; @state() private splitView = true; @state() filterPagesBy: { filterQABy?: string; gte?: number; gt?: number; lte?: number; lt?: number; reviewed?: boolean; approved?: boolean; hasNotes?: boolean; } = {}; @state() sortPagesBy: APISortQuery & { sortBy: SortableFieldNames } = { sortBy: "screenshotMatch", sortDirection: 1, }; private readonly api = new APIController(this); private readonly navigate = new NavigateController(this); private readonly notify = new NotifyController(this); private readonly replaySwReg = navigator.serviceWorker.getRegistration("/replay/"); @query("#replayframe") private readonly replayFrame?: HTMLIFrameElement | null; @query(".reviewDialog") private readonly reviewDialog?: Dialog | null; @query(".commentDialog") private readonly commentDialog?: Dialog | null; @query('sl-textarea[name="pageComment"]') private readonly commentTextarea?: SlTextarea | null; connectedCallback(): void { super.connectedCallback(); // Receive messages from replay-web-page windows void this.replaySwReg.then((reg) => { if (!reg) { console.log("[debug] no reg, listening to messages"); // window.addEventListener("message", this.onWindowMessage); } }); window.addEventListener("message", this.onWindowMessage); } disconnectedCallback(): void { super.disconnectedCallback(); if (this.crawlData?.blobUrl) URL.revokeObjectURL(this.crawlData.blobUrl); if (this.qaData?.blobUrl) URL.revokeObjectURL(this.qaData.blobUrl); window.removeEventListener("message", this.onWindowMessage); } private readonly onWindowMessage = (event: MessageEvent) => { const sourceLoc = (event.source as Window | null)?.location.href; // ensure its an rwp frame if (sourceLoc && sourceLoc.indexOf("?source=") > 0) { void this.handleRwpMessage(sourceLoc); } }; /** * Callback for when hidden RWP embeds are loaded and ready. * This won't fire if the RWP service worker is already * registered on the page due to RWP conditionally rendering * if the sw is not present. */ private async handleRwpMessage(sourceLoc: string) { console.log("[debug] handleRwpMessage", sourceLoc); // check if has /qa/ in path, then QA if (sourceLoc.indexOf("%2Fqa%2F") >= 0 && !this.qaDataRegistered) { this.qaDataRegistered = true; console.log("[debug] onWindowMessage qa", this.qaData); await this.fetchContentForTab({ qa: true }); await this.updateComplete; // otherwise main crawl replay } else if (!this.crawlDataRegistered) { this.crawlDataRegistered = true; console.log("[debug] onWindowMessage crawl", this.crawlData); await this.fetchContentForTab(); await this.updateComplete; } // if (this.crawlData && this.qaData) { // window.removeEventListener("message", this.onWindowMessage); // } } protected async willUpdate( changedProperties: PropertyValues | Map, ) { if (changedProperties.has("itemId") && this.itemId) { void this.initItem(); } else if ( changedProperties.has("filterPagesBy") || changedProperties.has("sortPagesBy") || changedProperties.has("qaRunId") ) { void this.fetchPages(); } if ( (changedProperties.has("itemPageId") || changedProperties.has("qaRunId")) && this.itemPageId ) { void this.fetchPage(); } // Re-fetch when tab, archived item page, or QA run ID changes // from an existing one, probably due to user interaction if (changedProperties.get("tab") || changedProperties.get("page")) { if (this.tab === "screenshots") { if (this.crawlData?.blobUrl) URL.revokeObjectURL(this.crawlData.blobUrl); if (this.qaData?.blobUrl) URL.revokeObjectURL(this.qaData.blobUrl); } // TODO prefetch content for other tabs? void this.fetchContentForTab(); void this.fetchContentForTab({ qa: true }); } else if (changedProperties.get("qaRunId")) { void this.fetchContentForTab({ qa: true }); } } private async initItem() { void this.fetchCrawl(); await this.fetchQARuns(); const searchParams = new URLSearchParams(window.location.search); if (this.qaRunId) { if (this.itemPageId) { void this.fetchPages({ page: 1 }); } else { await this.fetchPages({ page: 1 }); } } const firstQARun = this.finishedQARuns?.[0]; const firstPage = this.pages?.items[0]; if (!this.qaRunId && firstQARun) { searchParams.set("qaRunId", firstQARun.id); } if (!this.itemPageId && firstPage) { searchParams.set("itemPageId", firstPage.id); } this.navigate.to(`${window.location.pathname}?${searchParams.toString()}`); } /** * Get current page position with previous and next items */ private getPageListSliceByCurrent( pageId = this.itemPageId, ): [ ArchivedItemQAPage | undefined, ArchivedItemQAPage | undefined, ArchivedItemQAPage | undefined, ] { if (!pageId || !this.pages) { return [undefined, undefined, undefined]; } const pages = this.pages.items; const idx = pages.findIndex(({ id }) => id === pageId); return [pages[idx - 1], pages[idx], pages[idx + 1]]; } private navToPage(pageId: string) { const searchParams = new URLSearchParams(window.location.search); searchParams.set("itemPageId", pageId); this.navigate.to( `${window.location.pathname}?${searchParams.toString()}`, undefined, /* resetScroll: */ false, ); } render() { const crawlBaseUrl = `${this.navigate.orgBasePath}/items/crawl/${this.itemId}`; const searchParams = new URLSearchParams(window.location.search); const itemName = this.item ? renderName(this.item) : nothing; const [prevPage, currentPage, nextPage] = this.getPageListSliceByCurrent(); const currentQARun = this.finishedQARuns?.find( ({ id }) => id === this.qaRunId, ); const disableReview = !currentQARun || isActive(currentQARun.state); return html` ${this.renderHidden()}
${msg("Back")}

${msg("Review Archived Item")}

${itemName}

${when( this.finishedQARuns, (qaRuns) => html` ) => { const params = new URLSearchParams(searchParams); params.set("qaRunId", e.detail.item.id); this.navigate.to( `${window.location.pathname}?${params.toString()}`, ); }} > `, )}
${msg("Exit Review")} void this.reviewDialog?.show()} ?disabled=${disableReview} > ${msg("Finish Review")}

${ this.page?.title || html`${msg("No page title")}` }

${msg("Previous Page")} void this.commentDialog?.show()} @btrix-update-page-approval=${this.onUpdatePageApproval} > ${msg("Next Page")}
${this.renderPanelToolbar()} ${this.renderPanel()}

${msg("Pages")}

, ) => { const { page } = e.detail; void this.fetchPages({ page }); }} @btrix-qa-page-select=${(e: CustomEvent) => { this.navToPage(e.detail); }} @btrix-qa-filter-change=${( e: CustomEvent, ) => { this.filterPagesBy = { ...this.filterPagesBy, ...e.detail, }; }} @btrix-qa-sort-change=${(e: CustomEvent) => { this.sortPagesBy = { ...this.sortPagesBy, ...e.detail, }; }} >
${this.renderComments()}

this.commentDialog?.submit()} > ${msg("Submit Comment")}
${msg("Excellent!")}
${msg( "This archived item perfectly replicates the original pages.", )}
${msg("Good")}
${msg( "Looks and functions nearly the same as the original pages.", )}
${msg("Fair")}
${msg( "Similar to the original pages, but may be missing non-critical content or functionality.", )}
${msg("Poor")}
${msg( "Some similarities with the original pages, but missing critical content or functionality.", )}
${msg("Bad")}
${msg( "Missing all content and functionality from the original pages.", )}
${msg("Cancel")} this.reviewDialog?.submit()} > ${msg("Submit Review")}
`; } private renderHidden() { const iframe = (reg?: ServiceWorkerRegistration) => when(this.page, () => { const onLoad = reg ? () => { void this.fetchContentForTab(); void this.fetchContentForTab({ qa: true }); } : () => { console.debug("waiting for post message instead"); }; // Use iframe to access replay content // Use a 'non-existent' URL on purpose so that RWP itself is not rendered, // but we need a /replay iframe for proper fetch() to service worker return html` `; }); const rwp = (reg?: ServiceWorkerRegistration) => when( !reg || !this.crawlDataRegistered || !this.qaDataRegistered, () => html` `, ); return guard( [ this.replaySwReg, this.page, this.itemId, this.qaRunId, this.crawlDataRegistered, this.qaDataRegistered, ], () => until( this.replaySwReg.then((reg) => { return html`${iframe(reg)}${rwp(reg)}`; }), ), ); } private renderComments() { return html` ${when( this.page?.notes?.length, (commentCount) => html` ${msg(str`Comments (${commentCount.toLocaleString()})`)}
    ${this.page?.notes?.map( (comment) => html`
  • ${msg( str`${comment.userName} commented on ${formatISODateString( comment.created, { hour: undefined, minute: undefined, }, )}`, )}
    this.deletePageComment(comment.id)} >
    ${comment.text}
  • `, )}
`, )}
`; } private renderPanelToolbar() { const buttons = html` ${choose(this.tab, [ // [ // "replay", // () => html` //
// //
// `, // ], [ "screenshots", () => html`
`, ], ])} `; return html`
${buttons}
${this.page?.url || "http://"}
${when( this.page, (page) => html` `, )}
`; } private renderPanel() { // cache DOM for faster switching between tabs const choosePanel = () => { switch (this.tab) { case "screenshots": return renderScreenshots(this.crawlData, this.qaData, this.splitView); case "text": return renderText(this.crawlData, this.qaData); case "resources": return renderResources(this.crawlData, this.qaData); case "replay": return renderReplay(this.crawlData); default: break; } }; return html`
${cache(choosePanel())}
`; } private readonly renderRWP = ( rwpId: string, { qa, url }: { qa: boolean; url?: string }, ) => { if (!rwpId) return; const replaySource = `/api/orgs/${this.orgId}/crawls/${this.itemId}${qa ? `/qa/${rwpId}` : ""}/replay.json`; const headers = this.authState?.headers; const config = JSON.stringify({ headers }); console.log("[debug] rendering rwp", rwpId); return guard( [rwpId, this.page, this.authState], () => html` `, ); }; private readonly onTabNavClick = (e: MouseEvent) => { this.navigate.link(e, undefined, /* resetScroll: */ false); }; private async onUpdatePageApproval(e: CustomEvent) { const updated = e.detail; if (!this.page || this.page.id !== updated.id) return; this.page = merge(this.page, updated); const reviewStatusChanged = this.page.approved !== updated.approved; if (reviewStatusChanged) { void this.fetchPages(); } } private async fetchCrawl(): Promise { try { this.item = await this.getCrawl(); } catch { this.notify.toast({ message: msg("Sorry, couldn't retrieve archived item at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private navNextPage() { const [, , nextPage] = this.getPageListSliceByCurrent(); if (nextPage) { this.navToPage(nextPage.id); } } private navPrevPage() { const [prevPage] = this.getPageListSliceByCurrent(); if (prevPage) { this.navToPage(prevPage.id); } } private async onSubmitComment(e: SubmitEvent) { e.preventDefault(); const value = this.commentTextarea?.value; if (!value) return; void this.commentDialog?.hide(); try { const { data } = await this.api.fetch<{ data: ArchivedItemPageComment }>( `/orgs/${this.orgId}/crawls/${this.itemId}/pages/${this.itemPageId}/notes`, this.authState!, { method: "POST", body: JSON.stringify({ text: value }), }, ); const commentForm = this.commentDialog?.querySelector("form"); if (commentForm) { commentForm.reset(); } const comments = [...this.page!.notes!, data]; this.page = merge(this.page!, { notes: comments }); void this.fetchPages(); } catch (e: unknown) { void this.commentDialog?.show(); console.debug(e); this.notify.toast({ message: msg("Sorry, couldn't add comment at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async deletePageComment(commentId: string): Promise { try { await this.api.fetch( `/orgs/${this.orgId}/crawls/${this.itemId}/pages/${this.itemPageId}/notes/delete`, this.authState!, { method: "POST", body: JSON.stringify({ delete_list: [commentId] }), }, ); const comments = this.page!.notes!.filter(({ id }) => id !== commentId); this.page = merge(this.page!, { notes: comments }); void this.fetchPages(); } catch { this.notify.toast({ message: msg("Sorry, couldn't delete comment at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async fetchQARuns(): Promise { try { this.finishedQARuns = (await this.getQARuns()).filter(({ state }) => finishedCrawlStates.includes(state), ); } catch { this.notify.toast({ message: msg("Sorry, couldn't retrieve analysis runs at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async getQARuns(): Promise { return this.api.fetch( `/orgs/${this.orgId}/crawls/${this.itemId}/qa?skipFailed=true`, this.authState!, ); } private async getCrawl(): Promise { return this.api.fetch( `/orgs/${this.orgId}/crawls/${this.itemId}`, this.authState!, ); } private async fetchPage(): Promise { if (!this.itemPageId) return; try { this.page = await this.getPage(this.itemPageId); } catch { this.notify.toast({ message: msg("Sorry, couldn't retrieve page at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private resolveType(url: string, { mime = "", type }: PageResource) { if (type) { type = type.toLowerCase(); } // Map common mime types where important information would be lost // if we only use first half to more descriptive resource types if (type === "script" || mime.includes("javascript")) { return "javascript"; } if (type === "stylesheet" || mime.includes("css")) { return "stylesheet"; } if (type === "image") { return "image"; } if (type === "font") { return "font"; } if (type === "ping") { return "other"; } if (url.endsWith("favicon.ico")) { return "favicon"; } let path = ""; try { path = new URL(url).pathname; } catch (e) { // ignore } const ext = path.slice(path.lastIndexOf(".") + 1); if (type === "fetch" || type === "xhr") { if (IMG_EXTS.includes(ext)) { return "image"; } } if (mime.includes("json") || ext === "json") { return "json"; } if (mime.includes("pdf") || ext === "pdf") { return "pdf"; } if ( type === "document" || mime.includes("html") || ext === "html" || ext === "htm" ) { return "html"; } if (!mime) { return "other"; } return mime.split("/")[0]; } private async fetchContentForTab({ qa } = { qa: false }): Promise { const page = this.page; const tab = this.tab; const sourceId = qa ? this.qaRunId : this.itemId; const frameWindow = this.replayFrame?.contentWindow; if (!page || !sourceId || !frameWindow) { console.log( "[debug] no page replaId or frameWindow", page, sourceId, frameWindow, ); return; } if (qa && tab === "replay") { return; } const timestamp = page.ts?.split(".")[0].replace(/\D/g, ""); const pageUrl = page.url; const doLoad = async (isQA: boolean) => { const urlPrefix = tabToPrefix[tab]; const urlPart = `${timestamp}mp_/${urlPrefix ? `urn:${urlPrefix}:` : ""}${pageUrl}`; const url = `/replay/w/${sourceId}/${urlPart}`; // TODO check status code if (tab === "replay") { return { replayUrl: url }; } const resp = await frameWindow.fetch(url); //console.log("resp:", resp); if (!resp.ok) { throw resp.status; } if (tab === "screenshots") { const blob = await resp.blob(); const blobUrl = URL.createObjectURL(blob) || ""; return { blobUrl }; } else if (tab === "text") { const text = await resp.text(); return { text }; } else { // tab === "resources" const json = (await resp.json()) as { urls: PageResource[]; }; // console.log(json); const typeMap = new Map(); let good = 0, bad = 0; for (const [url, entry] of Object.entries(json.urls)) { const { status = 0, type, mime } = entry; const resType = this.resolveType(url, entry); // for debugging logResource(isQA, resType, url, type, mime, status); if (!typeMap.has(resType)) { if (status < 400) { typeMap.set(resType, { good: 1, bad: 0 }); good++; } else { typeMap.set(resType, { good: 0, bad: 1 }); bad++; } } else { const count = typeMap.get(resType); if (status < 400) { count!.good++; good++; } else { count!.bad++; bad++; } typeMap.set(resType, count!); } } typeMap.set("Total", { good, bad }); // const text = JSON.stringify( // Object.fromEntries(typeMap.entries()), // null, // 2, // ); return { resources: Object.fromEntries(typeMap.entries()) }; } return { text: "" }; }; try { const content = await doLoad(qa); if (qa) { this.qaData = { ...this.qaData, ...content, }; this.qaDataRegistered = true; } else { this.crawlData = { ...this.crawlData, ...content, }; this.crawlDataRegistered = true; } } catch (e: unknown) { console.log("[debug] error:", e); // check if this endpoint is registered, if not, ensure re-render if (e === 404) { let hasEndpoint = false; try { const resp = await frameWindow.fetch(`/replay/w/api/c/${sourceId}`); hasEndpoint = !!resp.ok; } catch (e) { hasEndpoint = false; } if (qa) { this.qaData = hasEndpoint ? {} : null; this.qaDataRegistered = hasEndpoint; } else { this.crawlData = hasEndpoint ? {} : null; this.crawlDataRegistered = hasEndpoint; } } } } private async getPage(pageId: string): Promise { return this.api.fetch( this.qaRunId ? `/orgs/${this.orgId}/crawls/${this.itemId}/qa/${this.qaRunId}/pages/${pageId}` : `/orgs/${this.orgId}/crawls/${this.itemId}/pages/${pageId}`, this.authState!, ); } private async fetchPages(params?: APIPaginationQuery): Promise { try { this.pages = await this.getPages({ page: params?.page ?? this.pages?.page ?? 1, pageSize: params?.pageSize ?? this.pages?.pageSize ?? DEFAULT_PAGE_SIZE, ...this.sortPagesBy, }); } catch { this.notify.toast({ message: msg("Sorry, couldn't retrieve pages at this time."), variant: "danger", icon: "exclamation-octagon", }); } } private async getPages( params?: APIPaginationQuery & APISortQuery & { reviewed?: boolean }, ): Promise> { const query = queryString.stringify( { ...this.filterPagesBy, ...params, }, { arrayFormat: "comma", }, ); return this.api.fetch>( `/orgs/${this.orgId}/crawls/${this.itemId ?? ""}/qa/${this.qaRunId ?? ""}/pages?${query}`, this.authState!, ); } private async onReviewSubmit(e: SubmitEvent) { e.preventDefault(); const form = e.currentTarget as HTMLFormElement; const params = serialize(form); if (!params.reviewStatus) { return; } try { const data = await this.api.fetch<{ updated: boolean }>( `/orgs/${this.orgId}/all-crawls/${this.itemId}`, this.authState!, { method: "PATCH", body: JSON.stringify({ reviewStatus: +params.reviewStatus, description: params.description, }), }, ); if (!data.updated) { throw data; } void this.reviewDialog?.hide(); this.navigate.to( `${this.navigate.orgBasePath}/items/crawl/${this.itemId}#qa`, ); this.notify.toast({ message: msg("Saved QA review."), variant: "success", icon: "check2-circle", }); } catch (e) { this.notify.toast({ message: msg("Sorry, couldn't submit QA review at this time."), variant: "danger", icon: "exclamation-octagon", }); } } } // leaving here for further debugging of resources function logResource( _isQA: boolean, _resType: string, _url: string, _type?: string, _mime?: string, _status = 0, ) { // console.log( // _isQA ? "replay" : "crawl", // _status >= 400 ? "bad" : "good", // _resType, // _type, // _mime, // _status, // _url, // ); }