Watch crawl from crawl detail page (#156)

closes #164
closes #134 

Co-authored-by: Ilya Kreymer <ikreymer@users.noreply.github.com>
This commit is contained in:
sua yoo 2022-03-02 18:08:08 -08:00 committed by GitHub
parent 51a573ef1f
commit 373c489b00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 456 additions and 76 deletions

View File

@ -24,6 +24,9 @@ import("./sign-up-form").then(({ SignUpForm }) => {
import("./not-found").then(({ NotFound }) => {
customElements.define("btrix-not-found", NotFound);
});
import("./screencast").then(({ Screencast: Screencast }) => {
customElements.define("btrix-screencast", Screencast);
});
customElements.define("btrix-alert", Alert);
customElements.define("btrix-input", Input);

View File

@ -0,0 +1,316 @@
import { LitElement, html, css } from "lit";
import { property, state } from "lit/decorators.js";
type Message = {
id: string; // page ID
};
type InitMessage = Message & {
msg: "init";
browsers: number;
width: number;
height: number;
};
type ScreencastMessage = Message & {
msg: "screencast";
url: string; // page URL
data: string; // base64 PNG data
};
type CloseMessage = Message & {
msg: "close";
};
/**
* Watch page crawl
*
* Usage example:
* ```ts
* <btrix-screencast
* archiveId=${archiveId}
* crawlId=${crawlId}
* ></btrix-screencast>
* ```
*/
export class Screencast extends LitElement {
static styles = css`
.wrapper {
position: relative;
}
.spinner {
text-align: center;
font-size: 2rem;
}
.container {
display: grid;
gap: 0.5rem;
}
.screen {
border: 1px solid var(--sl-color-neutral-100);
border-radius: var(--sl-border-radius-medium);
cursor: pointer;
transition: opacity 0.1s border-color 0.1s;
overflow: hidden;
}
.screen:hover {
opacity: 0.8;
border-color: var(--sl-color-neutral-300);
}
figure {
margin: 0;
}
figcaption {
flex: 1;
border-bottom-width: 1px;
border-bottom-color: var(--sl-panel-border-color);
color: var(--sl-color-neutral-600);
font-size: var(--sl-font-size-small);
padding: var(--sl-spacing-x-small);
}
figcaption,
.dialog-label {
display: block;
font-size: var(--sl-font-size-small);
line-height: 1;
/* Truncate: */
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.dialog-label {
max-width: 40em;
}
img {
display: block;
width: 100%;
height: auto;
box-shadow: 0;
outline: 0;
border: 0;
}
`;
@property({ type: String })
authToken?: string;
@property({ type: String })
archiveId?: string;
@property({ type: String })
crawlId?: string;
@property({ type: Array })
watchIPs: string[] = [];
@state()
private dataList: Array<ScreencastMessage> = [];
@state()
private isConnecting: boolean = false;
@state()
private focusedScreenData?: ScreencastMessage;
// Websocket connections
private wsMap: Map<string, WebSocket> = new Map();
// Page image data
private imageDataMap: Map<string, ScreencastMessage> = new Map();
private screenCount = 1;
private screenWidth = 640;
shouldUpdate(changedProperties: Map<string, any>) {
if (changedProperties.size === 1 && changedProperties.has("watchIPs")) {
// Check stringified value of IP list
return (
this.watchIPs.toString() !==
changedProperties.get("watchIPs").toString()
);
}
return true;
}
protected firstUpdated() {
this.isConnecting = true;
// Connect to websocket server
this.connectWs();
}
async updated(changedProperties: Map<string, any>) {
if (
changedProperties.get("archiveId") ||
changedProperties.get("crawlId") ||
changedProperties.get("watchIPs") ||
changedProperties.get("authToken")
) {
// Reconnect
this.disconnectWs();
this.connectWs();
}
}
disconnectedCallback() {
this.disconnectWs();
super.disconnectedCallback();
}
render() {
return html`
<div class="wrapper">
${this.isConnecting
? html`<div class="spinner">
<sl-spinner></sl-spinner>
</div> `
: ""}
<div
class="container"
style="grid-template-columns: repeat(${this
.screenCount}, minmax(0, 1fr))"
>
${this.dataList.map(
(pageData) => html` <figure
class="screen"
title="${pageData.url}"
role="button"
@click=${() => (this.focusedScreenData = pageData)}
>
<figcaption>${pageData.url}</figcaption>
<img src="data:image/png;base64,${pageData.data}" />
</figure>`
)}
</div>
</div>
<sl-dialog
?open=${Boolean(this.focusedScreenData)}
style="--width: ${this.screenWidth}px;
--header-spacing: var(--sl-spacing-small);
--body-spacing: 0;
"
@sl-after-hide=${this.unfocusScreen}
>
<span
class="dialog-label"
slot="label"
title=${this.focusedScreenData?.url || ""}
>
${this.focusedScreenData?.url}
</span>
${this.focusedScreenData
? html`
<img
src="data:image/png;base64,${this.focusedScreenData.data}"
title="${this.focusedScreenData.url}"
/>
`
: ""}
</sl-dialog>
`;
}
private connectWs() {
if (!this.archiveId || !this.crawlId) {
return;
}
if (!this.watchIPs?.length) {
console.warn("No watch IPs to connect to");
return;
}
const baseURL = `${window.location.protocol === "https:" ? "wss" : "ws"}:${
process.env.API_HOST
}/watch/${this.archiveId}/${this.crawlId}`;
this.watchIPs.forEach((ip: string) => {
const ws = new WebSocket(
`${baseURL}/${ip}/ws?auth_bearer=${this.authToken || ""}`
);
ws.addEventListener("open", () => {
if (this.wsMap.size === this.watchIPs.length) {
this.isConnecting = false;
}
});
ws.addEventListener("close", () => {
this.wsMap.delete(ip);
});
ws.addEventListener("error", () => {
this.isConnecting = false;
});
ws.addEventListener("message", ({ data }) => {
this.handleMessage(JSON.parse(data));
});
this.wsMap.set(ip, ws);
});
}
private disconnectWs() {
this.isConnecting = false;
this.wsMap.forEach((ws) => {
ws.close();
});
}
private handleMessage(
message: InitMessage | ScreencastMessage | CloseMessage
) {
if (message.msg === "init") {
this.screenCount = message.browsers;
this.screenWidth = message.width;
} else {
const { id } = message;
if (message.msg === "screencast") {
if (message.url === "about:blank") {
// Skip blank pages
return;
}
if (this.isConnecting) {
this.isConnecting = false;
}
this.imageDataMap.set(id, message);
if (this.focusedScreenData?.id === id) {
this.focusedScreenData = message;
}
this.updateDataList();
} else if (message.msg === "close") {
this.imageDataMap.delete(id);
this.updateDataList();
}
}
}
updateDataList() {
// keep same number of data entries (probably should only decrease if scale is reduced)
this.dataList = [
...this.imageDataMap.values(),
...this.dataList.slice(this.imageDataMap.size),
];
}
unfocusScreen() {
this.updateDataList();
this.focusedScreenData = undefined;
}
}

File diff suppressed because one or more lines are too long

View File

@ -147,11 +147,19 @@ export class App extends LiteElement {
}
navigate(newViewPath: string, state?: object) {
let url;
if (newViewPath.startsWith("http")) {
const url = new URL(newViewPath);
newViewPath = `${url.pathname}${url.search}`;
url = new URL(newViewPath);
} else {
url = new URL(
`${window.location.origin}/${newViewPath.replace(/^\//, "")}`
);
}
// Remove hash from path for matching
newViewPath = `${url.pathname}${url.search}`;
if (newViewPath === "/log-in" && this.authService.authState) {
// Redirect to logged in home page
this.viewState = this.router.match(DASHBOARD_ROUTE);
@ -161,7 +169,11 @@ export class App extends LiteElement {
this.viewState.data = state;
window.history.pushState(this.viewState, "", this.viewState.pathname);
window.history.pushState(
this.viewState,
"",
`${this.viewState.pathname}${url.hash}${url.search}`
);
}
navLink(event: Event) {

View File

@ -65,13 +65,6 @@ export class CrawlDetail extends LiteElement {
async firstUpdated() {
this.fetchCrawl();
// try {
// this.watchUrl = await this.watchCrawl();
// console.log(this.watchUrl);
// } catch (e) {
// console.error(e);
// }
}
connectedCallback(): void {
@ -92,9 +85,21 @@ export class CrawlDetail extends LiteElement {
let sectionContent: string | TemplateResult = "";
switch (this.sectionName) {
case "watch":
sectionContent = this.renderWatch();
case "watch": {
if (this.crawl) {
if (this.crawl.state === "running") {
sectionContent = this.renderWatch();
} else {
sectionContent = this.renderReplay();
}
} else {
// TODO loading indicator?
return "";
}
break;
}
case "download":
sectionContent = this.renderFiles();
break;
@ -384,6 +389,41 @@ export class CrawlDetail extends LiteElement {
}
private renderWatch() {
if (!this.authState) return "";
const authToken = this.authState.headers.Authorization.split(" ")[1];
return html`
<header class="flex justify-between">
<h3 class="text-lg font-medium mb-2">${msg("Watch Crawl")}</h3>
${document.fullscreenEnabled
? html`
<sl-icon-button
name="arrows-fullscreen"
label=${msg("Fullscreen")}
@click=${() => this.enterFullscreen("screencast-crawl")}
></sl-icon-button>
`
: ""}
</header>
${this.crawl
? html`
<div id="screencast-crawl">
<btrix-screencast
authToken=${authToken}
archiveId=${this.archiveId!}
crawlId=${this.crawlId!}
.watchIPs=${this.crawl.watchIPs || []}
></btrix-screencast>
</div>`
: ""}
`;
}
private renderReplay() {
const isRunning = this.crawl?.state === "running";
const bearer = this.authState?.headers?.Authorization?.split(" ", 2)[1];
// for now, just use the first file until multi-wacz support is fully implemented
@ -391,10 +431,22 @@ export class CrawlDetail extends LiteElement {
const replaySource = this.crawl?.resources?.[0]?.path;
return html`
<h3 class="text-lg font-medium my-2">${msg("Watch or Replay Crawl")}</h3>
<header class="flex justify-between">
<h3 class="text-lg font-medium mb-2">${msg("Replay Crawl")}</h3>
${document.fullscreenEnabled
? html`
<sl-icon-button
name="arrows-fullscreen"
label=${msg("Fullscreen")}
@click=${() => this.enterFullscreen("replay-crawl")}
></sl-icon-button>
`
: ""}
</header>
<div
class="aspect-video rounded border ${this.isRunning
id="replay-crawl"
class="aspect-4/3 rounded border ${isRunning
? "border-purple-200"
: "border-slate-100"}"
>
@ -408,38 +460,6 @@ export class CrawlDetail extends LiteElement {
></replay-web-page>`
: ``}
</div>
<div
class="absolute top-2 right-2 flex bg-white/90 hover:bg-white rounded-full"
>
${this.isWatchExpanded
? html`
<sl-icon-button
class="px-1"
name="arrows-angle-contract"
label=${msg("Contract crawl video")}
@click=${() => (this.isWatchExpanded = false)}
></sl-icon-button>
`
: html`
<sl-icon-button
class="px-1"
name="arrows-angle-expand"
label=${msg("Expand crawl video")}
@click=${() => (this.isWatchExpanded = true)}
></sl-icon-button>
`}
${this.watchUrl
? html`
<sl-icon-button
class="border-l px-1"
href=${this.watchUrl}
name="box-arrow-up-right"
label=${msg("Open in new window")}
target="_blank"
></sl-icon-button>
`
: ""}
</div>
`;
}
@ -664,18 +684,6 @@ export class CrawlDetail extends LiteElement {
return data;
}
private async watchCrawl(): Promise<string> {
const data = await this.apiFetch(
`/archives/${this.archiveId}/crawls/${this.crawlId}/watch`,
this.authState!,
{
method: "POST",
}
);
return data.watch_url;
}
private async cancel() {
if (window.confirm(msg("Are you sure you want to cancel the crawl?"))) {
const data = await this.apiFetch(
@ -761,6 +769,21 @@ export class CrawlDetail extends LiteElement {
private stopPollTimer() {
window.clearTimeout(this.timerId);
}
/**
* Enter fullscreen mode
* @param id ID of element to fullscreen
*/
private async enterFullscreen(id: string) {
try {
document.getElementById(id)!.requestFullscreen({
// Show browser navigation controls
navigationUI: "show",
});
} catch (err) {
console.error(err);
}
}
}
customElements.define("btrix-crawl-detail", CrawlDetail);

View File

@ -759,7 +759,7 @@ export class CrawlTemplatesDetail extends LiteElement {
${this.crawlTemplate.currCrawlId
? html` <a
class="text-primary font-medium hover:underline text-sm p-1"
href=${`/archives/${this.archiveId}/crawls/crawl/${this.crawlTemplate.currCrawlId}`}
href=${`/archives/${this.archiveId}/crawls/crawl/${this.crawlTemplate.currCrawlId}#watch`}
@click=${this.navLink}
>${msg("View crawl")}</a
>`
@ -787,7 +787,7 @@ export class CrawlTemplatesDetail extends LiteElement {
${this.crawlTemplate?.lastCrawlId
? html`<a
class="text-primary font-medium hover:underline text-sm p-1"
href=${`/archives/${this.archiveId}/crawls/crawl/${this.crawlTemplate.lastCrawlId}`}
href=${`/archives/${this.archiveId}/crawls/crawl/${this.crawlTemplate.lastCrawlId}#watch`}
@click=${this.navLink}
>${msg("View crawl")}</a
>
@ -1240,7 +1240,8 @@ export class CrawlTemplatesDetail extends LiteElement {
<br />
<a
class="underline hover:no-underline"
href="/archives/${this.archiveId}/crawls/crawl/${data.started}"
href="/archives/${this
.archiveId}/crawls/crawl/${data.started}#watch"
@click=${this.navLink.bind(this)}
>View crawl</a
>`

View File

@ -364,7 +364,7 @@ export class CrawlTemplatesList extends LiteElement {
? this.navTo(
`/archives/${this.archiveId}/crawls/crawl/${
this.runningCrawlsMap[t.id]
}`
}#watch`
)
: this.runNow(t);
}}
@ -502,7 +502,8 @@ export class CrawlTemplatesList extends LiteElement {
html`Started crawl from <strong>${template.name}</strong>. <br />
<a
class="underline hover:no-underline"
href="/archives/${this.archiveId}/crawls/crawl/${data.started}"
href="/archives/${this
.archiveId}/crawls/crawl/${data.started}#watch"
@click=${this.navLink.bind(this)}
>View crawl</a
>`

View File

@ -157,7 +157,8 @@ export class CrawlTemplatesList extends LiteElement {
html`Started crawl from <strong>${template.name}</strong>. <br />
<a
class="underline hover:no-underline"
href="/archives/${this.archiveId}/crawls/crawl/${data.started}"
href="/archives/${this
.archiveId}/crawls/crawl/${data.started}#watch"
@click=${this.navLink.bind(this)}
>View crawl</a
>`

View File

@ -53,7 +53,7 @@ export class Archive extends LiteElement {
isNewResourceTab: boolean = false;
@state()
private archive?: ArchiveData;
private archive?: ArchiveData | null;
@state()
private successfullyInvitedEmail?: string;
@ -61,14 +61,22 @@ export class Archive extends LiteElement {
async firstUpdated() {
if (!this.archiveId) return;
const archive = await this.getArchive(this.archiveId);
try {
const archive = await this.getArchive(this.archiveId);
if (!archive) {
this.navTo("/archives");
} else {
this.archive = archive;
if (!archive) {
this.navTo("/archives");
} else {
this.archive = archive;
}
} catch {
this.archive = null;
// TODO get archive members
this.notify({
message: msg("Sorry, couldn't retrieve archive at this time."),
type: "danger",
icon: "exclamation-octagon",
});
}
}
@ -79,6 +87,11 @@ export class Archive extends LiteElement {
}
render() {
if (this.archive === null) {
// TODO handle 404 and 500s
return "";
}
if (!this.archive) {
return html`
<div

View File

@ -15,6 +15,7 @@ export type Crawl = {
fileCount?: number;
fileSize?: number;
completions?: number;
watchIPs?: Array<string>;
};
type SeedConfig = {

View File

@ -47,6 +47,9 @@ function makeTheme() {
lg: `var(--sl-shadow-large)`,
xl: `var(--sl-shadow-x-large)`,
},
aspectRatio: {
"4/3": "4 / 3", // For Browsertrix watch/replay
},
};
}

View File

@ -1,9 +1,9 @@
// webpack.config.js
const path = require("path");
const webpack = require("webpack");
const ESLintPlugin = require("eslint-webpack-plugin");
const HtmlWebpackPlugin = require("html-webpack-plugin");
const CopyPlugin = require("copy-webpack-plugin");
const Dotenv = require("dotenv-webpack");
const childProcess = require("child_process");
const isDevServer = process.env.WEBPACK_SERVE;
@ -23,14 +23,16 @@ const execCommand = (cmd, defValue) => {
} catch (e) {
return defValue;
}
}
};
// Local dev only
// Git branch and commit hash is used to add build info to error reporter when running locally
const gitBranch = process.env.GIT_BRANCH_NAME ||
const gitBranch =
process.env.GIT_BRANCH_NAME ||
execCommand("git rev-parse --abbrev-ref HEAD", "unknown");
const commitHash = process.env.GIT_COMMIT_HASH ||
const commitHash =
process.env.GIT_COMMIT_HASH ||
execCommand("git rev-parse --short HEAD", "unknown");
require("dotenv").config({
@ -106,6 +108,7 @@ module.exports = {
Host: backendUrl.host,
},
pathRewrite: { "^/api": "" },
ws: true,
},
},
// Serve replay service worker file
@ -119,7 +122,9 @@ module.exports = {
},
plugins: [
new Dotenv({ path: dotEnvPath }),
new webpack.DefinePlugin({
"process.env.API_HOST": JSON.stringify(backendUrl.host),
}),
new HtmlWebpackPlugin({
template: "src/index.ejs",