import { localized, msg, str } from "@lit/localize";
import type { SlTextarea } from "@shoelace-style/shoelace";
import { serialize } from "@shoelace-style/shoelace/dist/utilities/form.js";
import { merge } from "immutable";
import { html, nothing, type PropertyValues } from "lit";
import { customElement, property, query, state } from "lit/decorators.js";
import { cache } from "lit/directives/cache.js";
import { choose } from "lit/directives/choose.js";
import { guard } from "lit/directives/guard.js";
import { ifDefined } from "lit/directives/if-defined.js";
import { until } from "lit/directives/until.js";
import { when } from "lit/directives/when.js";
import queryString from "query-string";
import { styles } from "./styles";
import type * as QATypes from "./types";
import { renderReplay } from "./ui/replay";
import { renderResources } from "./ui/resources";
import { renderScreenshots } from "./ui/screenshots";
import { renderSeverityBadge } from "./ui/severityBadge";
import { renderText } from "./ui/text";
import { TailwindElement } from "@/classes/TailwindElement";
import type { Dialog } from "@/components/ui/dialog";
import { APIController } from "@/controllers/api";
import { NavigateController } from "@/controllers/navigate";
import { NotifyController } from "@/controllers/notify";
import {
type QaFilterChangeDetail,
type QaPaginationChangeDetail,
type QaSortChangeDetail,
type SortableFieldNames,
type SortDirection,
} from "@/features/qa/page-list/page-list";
import { type UpdatePageApprovalDetail } from "@/features/qa/page-qa-approval";
import type { SelectDetail } from "@/features/qa/qa-run-dropdown";
import type {
APIPaginatedList,
APIPaginationQuery,
APISortQuery,
} from "@/types/api";
import type { ArchivedItem, ArchivedItemPageComment } from "@/types/crawler";
import type { ArchivedItemQAPage, QARun } from "@/types/qa";
import { type AuthState } from "@/utils/AuthService";
import { finishedCrawlStates, isActive, renderName } from "@/utils/crawler";
import { formatISODateString, getLocale } from "@/utils/localization";
const DEFAULT_PAGE_SIZE = 100;
type PageResource = {
status?: number;
mime?: string;
type?: string;
};
// From https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types
const IMG_EXTS = [
"apng",
"avif",
"gif",
"jpg",
"jpeg",
"jfif",
"pjpeg",
"pjp",
"png",
"svg",
"webp",
"tif",
"tiff",
"bmp",
"ico",
"cur",
];
const tabToPrefix: Record = {
screenshots: "view",
text: "text",
resources: "pageinfo",
replay: "",
};
@localized()
@customElement("btrix-archived-item-qa")
export class ArchivedItemQA extends TailwindElement {
static styles = styles;
@property({ type: Object })
authState?: AuthState;
@property({ type: String })
orgId?: string;
@property({ type: String })
itemId?: string;
@property({ type: String })
itemPageId?: string;
@property({ type: String })
qaRunId?: string;
@property({ type: String })
tab: QATypes.QATab = "screenshots";
@state()
private item?: ArchivedItem;
@state()
finishedQARuns:
| (QARun & { state: (typeof finishedCrawlStates)[number] })[]
| undefined = [];
@state()
private pages?: APIPaginatedList;
@property({ type: Object })
page?: ArchivedItemQAPage;
@state()
private crawlData: QATypes.ReplayData = null;
@state()
private qaData: QATypes.ReplayData = null;
// indicate whether the crawl / qa endpoints have been registered in SW
// if not, requires loading via
// endpoints may be registered but crawlData / qaData may still be missing
@state()
private crawlDataRegistered = false;
@state()
private qaDataRegistered = false;
@state()
private splitView = true;
@state()
filterPagesBy: {
filterQABy?: string;
gte?: number;
gt?: number;
lte?: number;
lt?: number;
reviewed?: boolean;
approved?: boolean;
hasNotes?: boolean;
} = {};
@state()
sortPagesBy: APISortQuery & { sortBy: SortableFieldNames } = {
sortBy: "screenshotMatch",
sortDirection: 1,
};
private readonly api = new APIController(this);
private readonly navigate = new NavigateController(this);
private readonly notify = new NotifyController(this);
private readonly replaySwReg =
navigator.serviceWorker.getRegistration("/replay/");
@query("#replayframe")
private readonly replayFrame?: HTMLIFrameElement | null;
@query(".reviewDialog")
private readonly reviewDialog?: Dialog | null;
@query(".commentDialog")
private readonly commentDialog?: Dialog | null;
@query('sl-textarea[name="pageComment"]')
private readonly commentTextarea?: SlTextarea | null;
connectedCallback(): void {
super.connectedCallback();
// Receive messages from replay-web-page windows
void this.replaySwReg.then((reg) => {
if (!reg) {
console.log("[debug] no reg, listening to messages");
// window.addEventListener("message", this.onWindowMessage);
}
});
window.addEventListener("message", this.onWindowMessage);
}
disconnectedCallback(): void {
super.disconnectedCallback();
if (this.crawlData?.blobUrl) URL.revokeObjectURL(this.crawlData.blobUrl);
if (this.qaData?.blobUrl) URL.revokeObjectURL(this.qaData.blobUrl);
window.removeEventListener("message", this.onWindowMessage);
}
private readonly onWindowMessage = (event: MessageEvent) => {
const sourceLoc = (event.source as Window | null)?.location.href;
// ensure its an rwp frame
if (sourceLoc && sourceLoc.indexOf("?source=") > 0) {
void this.handleRwpMessage(sourceLoc);
}
};
/**
* Callback for when hidden RWP embeds are loaded and ready.
* This won't fire if the RWP service worker is already
* registered on the page due to RWP conditionally rendering
* if the sw is not present.
*/
private async handleRwpMessage(sourceLoc: string) {
console.log("[debug] handleRwpMessage", sourceLoc);
// check if has /qa/ in path, then QA
if (sourceLoc.indexOf("%2Fqa%2F") >= 0 && !this.qaDataRegistered) {
this.qaDataRegistered = true;
console.log("[debug] onWindowMessage qa", this.qaData);
await this.fetchContentForTab({ qa: true });
await this.updateComplete;
// otherwise main crawl replay
} else if (!this.crawlDataRegistered) {
this.crawlDataRegistered = true;
console.log("[debug] onWindowMessage crawl", this.crawlData);
await this.fetchContentForTab();
await this.updateComplete;
}
// if (this.crawlData && this.qaData) {
// window.removeEventListener("message", this.onWindowMessage);
// }
}
protected async willUpdate(
changedProperties: PropertyValues | Map,
) {
if (changedProperties.has("itemId") && this.itemId) {
void this.initItem();
} else if (
changedProperties.has("filterPagesBy") ||
changedProperties.has("sortPagesBy") ||
changedProperties.has("qaRunId")
) {
void this.fetchPages();
}
if (
(changedProperties.has("itemPageId") ||
changedProperties.has("qaRunId")) &&
this.itemPageId
) {
void this.fetchPage();
}
// Re-fetch when tab, archived item page, or QA run ID changes
// from an existing one, probably due to user interaction
if (changedProperties.get("tab") || changedProperties.get("page")) {
if (this.tab === "screenshots") {
if (this.crawlData?.blobUrl)
URL.revokeObjectURL(this.crawlData.blobUrl);
if (this.qaData?.blobUrl) URL.revokeObjectURL(this.qaData.blobUrl);
}
// TODO prefetch content for other tabs?
void this.fetchContentForTab();
void this.fetchContentForTab({ qa: true });
} else if (changedProperties.get("qaRunId")) {
void this.fetchContentForTab({ qa: true });
}
}
private async initItem() {
void this.fetchCrawl();
await this.fetchQARuns();
const searchParams = new URLSearchParams(window.location.search);
if (this.qaRunId) {
if (this.itemPageId) {
void this.fetchPages({ page: 1 });
} else {
await this.fetchPages({ page: 1 });
}
}
const firstQARun = this.finishedQARuns?.[0];
const firstPage = this.pages?.items[0];
if (!this.qaRunId && firstQARun) {
searchParams.set("qaRunId", firstQARun.id);
}
if (!this.itemPageId && firstPage) {
searchParams.set("itemPageId", firstPage.id);
}
this.navigate.to(`${window.location.pathname}?${searchParams.toString()}`);
}
/**
* Get current page position with previous and next items
*/
private getPageListSliceByCurrent(
pageId = this.itemPageId,
): [
ArchivedItemQAPage | undefined,
ArchivedItemQAPage | undefined,
ArchivedItemQAPage | undefined,
] {
if (!pageId || !this.pages) {
return [undefined, undefined, undefined];
}
const pages = this.pages.items;
const idx = pages.findIndex(({ id }) => id === pageId);
return [pages[idx - 1], pages[idx], pages[idx + 1]];
}
private navToPage(pageId: string) {
const searchParams = new URLSearchParams(window.location.search);
searchParams.set("itemPageId", pageId);
this.navigate.to(
`${window.location.pathname}?${searchParams.toString()}`,
undefined,
/* resetScroll: */ false,
);
}
render() {
const crawlBaseUrl = `${this.navigate.orgBasePath}/items/crawl/${this.itemId}`;
const searchParams = new URLSearchParams(window.location.search);
const itemName = this.item ? renderName(this.item) : nothing;
const [prevPage, currentPage, nextPage] = this.getPageListSliceByCurrent();
const currentQARun = this.finishedQARuns?.find(
({ id }) => id === this.qaRunId,
);
const disableReview = !currentQARun || isActive(currentQARun.state);
return html`
${this.renderHidden()}
${
this.page?.title ||
html`${msg("No page title")}`
}
${msg("Previous Page")}
void this.commentDialog?.show()}
@btrix-update-page-approval=${this.onUpdatePageApproval}
>
${msg("Next Page")}
${this.renderPanelToolbar()} ${this.renderPanel()}
${msg("Pages")}
,
) => {
const { page } = e.detail;
void this.fetchPages({ page });
}}
@btrix-qa-page-select=${(e: CustomEvent) => {
this.navToPage(e.detail);
}}
@btrix-qa-filter-change=${(
e: CustomEvent,
) => {
this.filterPagesBy = {
...this.filterPagesBy,
...e.detail,
};
}}
@btrix-qa-sort-change=${(e: CustomEvent) => {
this.sortPagesBy = {
...this.sortPagesBy,
...e.detail,
};
}}
>
this.commentDialog?.submit()}
>
${msg("Submit Comment")}
${msg("Cancel")}
this.reviewDialog?.submit()}
>
${msg("Submit Review")}
`;
}
private renderHidden() {
const iframe = (reg?: ServiceWorkerRegistration) =>
when(this.page, () => {
const onLoad = reg
? () => {
void this.fetchContentForTab();
void this.fetchContentForTab({ qa: true });
}
: () => {
console.debug("waiting for post message instead");
};
// Use iframe to access replay content
// Use a 'non-existent' URL on purpose so that RWP itself is not rendered,
// but we need a /replay iframe for proper fetch() to service worker
return html`
`;
});
const rwp = (reg?: ServiceWorkerRegistration) =>
when(
!reg || !this.crawlDataRegistered || !this.qaDataRegistered,
() => html`
${this.itemId && !this.crawlDataRegistered
? this.renderRWP(this.itemId, { qa: false })
: nothing}
${this.qaRunId && !this.qaDataRegistered
? this.renderRWP(this.qaRunId, { qa: true })
: nothing}
`,
);
return guard(
[
this.replaySwReg,
this.page,
this.itemId,
this.qaRunId,
this.crawlDataRegistered,
this.qaDataRegistered,
],
() =>
until(
this.replaySwReg.then((reg) => {
return html`${iframe(reg)}${rwp(reg)}`;
}),
),
);
}
private renderComments() {
return html`
${when(
this.page?.notes?.length,
(commentCount) => html`
${msg(str`Comments (${commentCount.toLocaleString()})`)}
${this.page?.notes?.map(
(comment) =>
html`-
${msg(
str`${comment.userName} commented on ${formatISODateString(
comment.created,
{
hour: undefined,
minute: undefined,
},
)}`,
)}
this.deletePageComment(comment.id)}
>
${comment.text}
`,
)}
`,
)}
`;
}
private renderPanelToolbar() {
const buttons = html`
${choose(this.tab, [
// [
// "replay",
// () => html`
//
//
//
// `,
// ],
[
"screenshots",
() => html`
(this.splitView = !this.splitView)}"
>
`,
],
])}
`;
return html`
${buttons}
${this.page?.url || "http://"}
${when(
this.page,
(page) => html`
`,
)}
`;
}
private renderPanel() {
// cache DOM for faster switching between tabs
const choosePanel = () => {
switch (this.tab) {
case "screenshots":
return renderScreenshots(this.crawlData, this.qaData, this.splitView);
case "text":
return renderText(this.crawlData, this.qaData);
case "resources":
return renderResources(this.crawlData, this.qaData);
case "replay":
return renderReplay(this.crawlData);
default:
break;
}
};
return html`
`;
}
private readonly renderRWP = (
rwpId: string,
{ qa, url }: { qa: boolean; url?: string },
) => {
if (!rwpId) return;
const replaySource = `/api/orgs/${this.orgId}/crawls/${this.itemId}${qa ? `/qa/${rwpId}` : ""}/replay.json`;
const headers = this.authState?.headers;
const config = JSON.stringify({ headers });
console.log("[debug] rendering rwp", rwpId);
return guard(
[rwpId, this.page, this.authState],
() => html`
`,
);
};
private readonly onTabNavClick = (e: MouseEvent) => {
this.navigate.link(e, undefined, /* resetScroll: */ false);
};
private async onUpdatePageApproval(e: CustomEvent) {
const updated = e.detail;
if (!this.page || this.page.id !== updated.id) return;
this.page = merge(this.page, updated);
const reviewStatusChanged = this.page.approved !== updated.approved;
if (reviewStatusChanged) {
void this.fetchPages();
}
}
private async fetchCrawl(): Promise {
try {
this.item = await this.getCrawl();
} catch {
this.notify.toast({
message: msg("Sorry, couldn't retrieve archived item at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private navNextPage() {
const [, , nextPage] = this.getPageListSliceByCurrent();
if (nextPage) {
this.navToPage(nextPage.id);
}
}
private navPrevPage() {
const [prevPage] = this.getPageListSliceByCurrent();
if (prevPage) {
this.navToPage(prevPage.id);
}
}
private async onSubmitComment(e: SubmitEvent) {
e.preventDefault();
const value = this.commentTextarea?.value;
if (!value) return;
void this.commentDialog?.hide();
try {
const { data } = await this.api.fetch<{ data: ArchivedItemPageComment }>(
`/orgs/${this.orgId}/crawls/${this.itemId}/pages/${this.itemPageId}/notes`,
this.authState!,
{
method: "POST",
body: JSON.stringify({ text: value }),
},
);
const commentForm = this.commentDialog?.querySelector("form");
if (commentForm) {
commentForm.reset();
}
const comments = [...this.page!.notes!, data];
this.page = merge(this.page!, { notes: comments });
void this.fetchPages();
} catch (e: unknown) {
void this.commentDialog?.show();
console.debug(e);
this.notify.toast({
message: msg("Sorry, couldn't add comment at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private async deletePageComment(commentId: string): Promise {
try {
await this.api.fetch(
`/orgs/${this.orgId}/crawls/${this.itemId}/pages/${this.itemPageId}/notes/delete`,
this.authState!,
{
method: "POST",
body: JSON.stringify({ delete_list: [commentId] }),
},
);
const comments = this.page!.notes!.filter(({ id }) => id !== commentId);
this.page = merge(this.page!, { notes: comments });
void this.fetchPages();
} catch {
this.notify.toast({
message: msg("Sorry, couldn't delete comment at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private async fetchQARuns(): Promise {
try {
this.finishedQARuns = (await this.getQARuns()).filter(({ state }) =>
finishedCrawlStates.includes(state),
);
} catch {
this.notify.toast({
message: msg("Sorry, couldn't retrieve analysis runs at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private async getQARuns(): Promise {
return this.api.fetch(
`/orgs/${this.orgId}/crawls/${this.itemId}/qa?skipFailed=true`,
this.authState!,
);
}
private async getCrawl(): Promise {
return this.api.fetch(
`/orgs/${this.orgId}/crawls/${this.itemId}`,
this.authState!,
);
}
private async fetchPage(): Promise {
if (!this.itemPageId) return;
try {
this.page = await this.getPage(this.itemPageId);
} catch {
this.notify.toast({
message: msg("Sorry, couldn't retrieve page at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private resolveType(url: string, { mime = "", type }: PageResource) {
if (type) {
type = type.toLowerCase();
}
// Map common mime types where important information would be lost
// if we only use first half to more descriptive resource types
if (type === "script" || mime.includes("javascript")) {
return "javascript";
}
if (type === "stylesheet" || mime.includes("css")) {
return "stylesheet";
}
if (type === "image") {
return "image";
}
if (type === "font") {
return "font";
}
if (type === "ping") {
return "other";
}
if (url.endsWith("favicon.ico")) {
return "favicon";
}
let path = "";
try {
path = new URL(url).pathname;
} catch (e) {
// ignore
}
const ext = path.slice(path.lastIndexOf(".") + 1);
if (type === "fetch" || type === "xhr") {
if (IMG_EXTS.includes(ext)) {
return "image";
}
}
if (mime.includes("json") || ext === "json") {
return "json";
}
if (mime.includes("pdf") || ext === "pdf") {
return "pdf";
}
if (
type === "document" ||
mime.includes("html") ||
ext === "html" ||
ext === "htm"
) {
return "html";
}
if (!mime) {
return "other";
}
return mime.split("/")[0];
}
private async fetchContentForTab({ qa } = { qa: false }): Promise {
const page = this.page;
const tab = this.tab;
const sourceId = qa ? this.qaRunId : this.itemId;
const frameWindow = this.replayFrame?.contentWindow;
if (!page || !sourceId || !frameWindow) {
console.log(
"[debug] no page replaId or frameWindow",
page,
sourceId,
frameWindow,
);
return;
}
if (qa && tab === "replay") {
return;
}
const timestamp = page.ts?.split(".")[0].replace(/\D/g, "");
const pageUrl = page.url;
const doLoad = async (isQA: boolean) => {
const urlPrefix = tabToPrefix[tab];
const urlPart = `${timestamp}mp_/${urlPrefix ? `urn:${urlPrefix}:` : ""}${pageUrl}`;
const url = `/replay/w/${sourceId}/${urlPart}`;
// TODO check status code
if (tab === "replay") {
return { replayUrl: url };
}
const resp = await frameWindow.fetch(url);
//console.log("resp:", resp);
if (!resp.ok) {
throw resp.status;
}
if (tab === "screenshots") {
const blob = await resp.blob();
const blobUrl = URL.createObjectURL(blob) || "";
return { blobUrl };
} else if (tab === "text") {
const text = await resp.text();
return { text };
} else {
// tab === "resources"
const json = (await resp.json()) as {
urls: PageResource[];
};
// console.log(json);
const typeMap = new Map();
let good = 0,
bad = 0;
for (const [url, entry] of Object.entries(json.urls)) {
const { status = 0, type, mime } = entry;
const resType = this.resolveType(url, entry);
// for debugging
logResource(isQA, resType, url, type, mime, status);
if (!typeMap.has(resType)) {
if (status < 400) {
typeMap.set(resType, { good: 1, bad: 0 });
good++;
} else {
typeMap.set(resType, { good: 0, bad: 1 });
bad++;
}
} else {
const count = typeMap.get(resType);
if (status < 400) {
count!.good++;
good++;
} else {
count!.bad++;
bad++;
}
typeMap.set(resType, count!);
}
}
typeMap.set("Total", { good, bad });
// const text = JSON.stringify(
// Object.fromEntries(typeMap.entries()),
// null,
// 2,
// );
return { resources: Object.fromEntries(typeMap.entries()) };
}
return { text: "" };
};
try {
const content = await doLoad(qa);
if (qa) {
this.qaData = {
...this.qaData,
...content,
};
this.qaDataRegistered = true;
} else {
this.crawlData = {
...this.crawlData,
...content,
};
this.crawlDataRegistered = true;
}
} catch (e: unknown) {
console.log("[debug] error:", e);
// check if this endpoint is registered, if not, ensure re-render
if (e === 404) {
let hasEndpoint = false;
try {
const resp = await frameWindow.fetch(`/replay/w/api/c/${sourceId}`);
hasEndpoint = !!resp.ok;
} catch (e) {
hasEndpoint = false;
}
if (qa) {
this.qaData = hasEndpoint ? {} : null;
this.qaDataRegistered = hasEndpoint;
} else {
this.crawlData = hasEndpoint ? {} : null;
this.crawlDataRegistered = hasEndpoint;
}
}
}
}
private async getPage(pageId: string): Promise {
return this.api.fetch(
this.qaRunId
? `/orgs/${this.orgId}/crawls/${this.itemId}/qa/${this.qaRunId}/pages/${pageId}`
: `/orgs/${this.orgId}/crawls/${this.itemId}/pages/${pageId}`,
this.authState!,
);
}
private async fetchPages(params?: APIPaginationQuery): Promise {
try {
this.pages = await this.getPages({
page: params?.page ?? this.pages?.page ?? 1,
pageSize: params?.pageSize ?? this.pages?.pageSize ?? DEFAULT_PAGE_SIZE,
...this.sortPagesBy,
});
} catch {
this.notify.toast({
message: msg("Sorry, couldn't retrieve pages at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private async getPages(
params?: APIPaginationQuery & APISortQuery & { reviewed?: boolean },
): Promise> {
const query = queryString.stringify(
{
...this.filterPagesBy,
...params,
},
{
arrayFormat: "comma",
},
);
return this.api.fetch>(
`/orgs/${this.orgId}/crawls/${this.itemId ?? ""}/qa/${this.qaRunId ?? ""}/pages?${query}`,
this.authState!,
);
}
private async onReviewSubmit(e: SubmitEvent) {
e.preventDefault();
const form = e.currentTarget as HTMLFormElement;
const params = serialize(form);
if (!params.reviewStatus) {
return;
}
try {
const data = await this.api.fetch<{ updated: boolean }>(
`/orgs/${this.orgId}/all-crawls/${this.itemId}`,
this.authState!,
{
method: "PATCH",
body: JSON.stringify({
reviewStatus: +params.reviewStatus,
description: params.description,
}),
},
);
if (!data.updated) {
throw data;
}
void this.reviewDialog?.hide();
this.navigate.to(
`${this.navigate.orgBasePath}/items/crawl/${this.itemId}#qa`,
);
this.notify.toast({
message: msg("Saved QA review."),
variant: "success",
icon: "check2-circle",
});
} catch (e) {
this.notify.toast({
message: msg("Sorry, couldn't submit QA review at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
}
// leaving here for further debugging of resources
function logResource(
_isQA: boolean,
_resType: string,
_url: string,
_type?: string,
_mime?: string,
_status = 0,
) {
// console.log(
// _isQA ? "replay" : "crawl",
// _status >= 400 ? "bad" : "good",
// _resType,
// _type,
// _mime,
// _status,
// _url,
// );
}